def test_http_status_code_to_type(self):
__all_http_status_codes__ = list(map(int, HTTPStatus))
for http_status_code in __all_http_status_codes__:
assert(pyhttpstatus_utils.get_http_status_type(http_status_code))
assert(type(pyhttpstatus_utils.is_http_status_successful(http_status_code)) is bool)
if pyhttpstatus_utils.is_http_status_successful(http_status_code):
assert(pyhttpstatus_utils.get_http_status_type(
http_status_code) == pyhttpstatus_utils.HttpStatusType.SUCCESSFUL)
else:
assert(pyhttpstatus_utils.get_http_status_type(
http_status_code) != pyhttpstatus_utils.HttpStatusType.SUCCESSFUL)
def get_http_responses_4xx_5xx():
client_error_type = get_http_status_type(400)
server_error_type = get_http_status_type(500)
ignore_list = [419, 425,
509] # Uncommon http responses excluded from the test
# Collect all possible 4xx and 5xx http responses
http_responses = [
code for code in HTTP_STATUS_PHRASE_DICT if get_http_status_type(code)
in [client_error_type, server_error_type]
]
http_responses = [
code for code in http_responses if code not in ignore_list
]
return http_responses
def tune_v2_request_retry_func(self, response):
"""Request Retry Function
Args:
response:
Returns:
Boolean
"""
try:
response_json = response.json()
except Exception as ex:
# No JSON response available.
raise TuneReportingError(
error_message='Invalid JSON response: {}'.format(response.text),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_JSON_DECODING
)
self.logger.debug("TMC API V2: Check for Retry: Start", extra=response_json)
tune_v2_status_code = None
tune_v2_errors_messages = ""
if 'status_code' in response_json:
tune_v2_status_code = response_json['status_code']
if 'errors' in response_json:
tune_v2_errors = response_json['errors']
if isinstance(tune_v2_errors, dict) \
and 'message' in tune_v2_errors:
tune_v2_errors_messages += tune_v2_errors['message']
elif isinstance(tune_v2_errors, list):
for tune_v2_error in tune_v2_errors:
if isinstance(tune_v2_error, dict) \
and 'message' in tune_v2_error:
tune_v2_errors_messages += tune_v2_error['message']
tune_v2_status_type = get_http_status_type(tune_v2_status_code)
response_extra = {
'status_code': tune_v2_status_code,
'status_type': tune_v2_status_type,
}
if tune_v2_errors_messages:
response_extra.update({'error_messages': safe_str(tune_v2_errors_messages)})
self.logger.debug("TMC API Base: Check for Retry: Response", extra=response_extra)
if tune_v2_status_code == 200:
self.logger.debug("TMC API Base: Check for Retry: Success", extra=response_extra)
return False
if tune_v2_status_code in [401, 403]:
self.logger.error("TMC API: Request: Error", extra={'status_code': tune_v2_status_code})
raise TuneRequestError(
error_message=tune_v2_errors_messages,
error_code=tune_v2_status_code,
)
if tune_v2_status_code in [404, 500]:
if "Api key was not found." in tune_v2_errors_messages:
self.logger.error("TMC API: Request: Error", extra={'tune_v2_status_code': tune_v2_status_code})
raise TuneRequestError(
error_message=tune_v2_errors_messages,
error_code=tune_v2_status_code,
)
self.logger.warning("TMC API Base: Check for Retry: Retry Candidate", extra=response_extra)
return True
self.logger.warning("TMC API Base: Check for Retry: No Retry", extra=response_extra)
return False
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
request_cert=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
request_cert: (optional) Cert tuple to enable Client side certificates.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise RequestsFortifiedValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise RequestsFortifiedValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
"{0}: Session: Details".format(request_label),
extra={
'cookie_payload': self.requests_session_client.session.cookies.get_dict(),
'request_label': request_label
}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug("{0}: Details".format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if request_cert:
kwargs.update({'cert': request_cert})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.note(
"{0}: Curl".format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug(
"{0}: {1}".format(request_label, request_method),
extra={'response_url': response.url}
)
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.requests_session_client.request(**kwargs)
except Exception as ex:
self.logger.error(
"{0}: Request Base: Error".format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
"{0}: Response: Failed".format(request_label),
extra={
'request_curl': self.built_request_curl
},
)
raise RequestsFortifiedModuleError(
error_message="{0}: Response: Failed".format(request_label),
error_code=RequestsFortifiedErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
"{0}: Response: Details".format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
"{0}: Cookie Payload".format(request_label),
extra={
'cookie_payload': self.requests_session_client.session.cookies.get_dict(),
'request_label': request_label
}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error("{0}: Response: Failed".format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error("{0}: Error: Response: Details".format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise RequestsFortifiedClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise RequestsFortifiedServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error("{0}: Error: Unhandled".format(request_label), extra=extra_unhandled)
raise RequestsFortifiedModuleError(**kwargs)
def request(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_retry=None,
request_retry_excps=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_cert=None,
cookie_payload=None,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False,
request_label=None
):
"""Request data from remote source with retries.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_retry: (optional) Retry configuration.
request_retry_func: (optional) Retry function, alternative
to request_retry_excps.
request_retry_excps: An exception or a tuple of exceptions
to catch.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
request_label:
Returns:
requests.Response: Data result if success or None if error.
Raises:
ServiceGatewayTimeoutError: Upon any timeout condition.
Exception: Upon error within this request_method.
Notes:
* tries: the maximum number of attempts. default: 1.
* delay: initial delay between attempts. default: 1.
* max_delay: the maximum value of delay. default: None (no limit).
* backoff: multiplier applied to delay between attempts.
default: 1 (no backoff).
* jitter: extra seconds added to delay between attempts.
default: 0.
"""
if request_label is None:
request_label = 'Request'
self.logger.debug(
"{0}: Start".format(request_label)
)
timeout = None
if not verify:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
if request_method:
request_method = request_method.upper()
if request_label is None:
request_label = 'Request'
if not request_retry:
request_retry = {}
if 'timeout' not in request_retry:
request_retry['timeout'] = self._REQUEST_CONFIG['timeout']
if 'tries' not in request_retry:
request_retry['tries'] = self._REQUEST_CONFIG['tries']
if 'delay' not in request_retry:
request_retry['delay'] = self._REQUEST_CONFIG['delay']
self._prep_request_retry(request_retry, request_retry_http_status_codes)
if not self.requests_session_client:
self.requests_session_client = RequestsSessionClient(
retry_tries=self.retry_tries,
retry_backoff=self.retry_backoff,
retry_codes=self.request_retry_http_status_codes
)
key_user_agent = 'User-Agent'
header_user_agent = {key_user_agent: __USER_AGENT__}
if request_headers:
if key_user_agent not in request_headers:
request_headers.update(header_user_agent)
else:
request_headers = header_user_agent
kwargs = {
'request_method': request_method,
'request_url': request_url,
'request_params': request_params,
'request_data': request_data,
'request_json': request_json,
'request_headers': request_headers,
'request_auth': request_auth,
'request_cert': request_cert,
'cookie_payload': cookie_payload,
'request_label': request_label,
'timeout': timeout,
'build_request_curl': build_request_curl,
'allow_redirects': allow_redirects,
'verify': verify,
'stream': stream
}
time_start_req = dt.datetime.now()
if request_retry_func is not None:
self.request_retry_func = request_retry_func
if request_retry_excps_func is not None:
self.request_retry_excps_func = request_retry_excps_func
if request_retry_http_status_codes is not None:
self.request_retry_http_status_codes = request_retry_http_status_codes
if request_retry_excps is not None:
self.request_retry_excps = request_retry_excps
extra_request = copy.copy(kwargs)
if request_label:
extra_request.update({'request_label': request_label})
if request_retry:
extra_request.update({'request_retry': request_retry})
if request_retry_func:
extra_request.update({'request_retry_func': request_retry_func})
if request_retry_http_status_codes:
extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes})
if request_retry_excps:
extra_request.update({'request_retry_excps': request_retry_excps})
if request_retry_excps:
extra_request.update({'request_retry_excps_func': request_retry_excps_func})
extra_request.update(env_usage())
self.logger.debug("{0}: Start: Details".format(request_label), extra=extra_request)
try:
self._prep_request_retry(request_retry, request_retry_http_status_codes)
response = self._request_retry(
call_func=self._request_data,
fargs=None,
fkwargs=kwargs,
timeout=timeout,
request_label=request_label,
request_retry_func=request_retry_func,
request_retry_excps_func=request_retry_excps_func
)
except (
requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
) as ex_req_timeout:
raise RequestsFortifiedServiceError(
error_message="{0}: Exception: Timeout".format(request_label),
errors=ex_req_timeout,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.HTTPError as ex_req_http:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: HTTPError".format(request_label),
errors=ex_req_http,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_HTTP
)
except (
requests.exceptions.ConnectionError,
) as ex_req_connect:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: ConnectionError".format(request_label),
errors=ex_req_connect,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (
requests.exceptions.ProxyError,
) as ex_req_proxy:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: ProxyError".format(request_label),
errors=ex_req_proxy,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (
requests.exceptions.SSLError,
) as ex_req_ssl:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: SSLError".format(request_label),
errors=ex_req_ssl,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (BrokenPipeError) as ex_broken_pipe:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: BrokenPipeError".format(request_label),
errors=ex_broken_pipe,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_CONNECT
)
except (ConnectionError) as ex_connect:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: ConnectionError".format(request_label),
errors=ex_connect,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_CONNECT
)
except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: ProtocolError".format(request_label),
errors=ex_req_urllib3_protocol,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout:
raise RequestsFortifiedServiceError(
error_message="{0}: Exception: Requests: ReadTimeoutError".format(request_label),
errors=ex_req_urllib3_read_timeout,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.TooManyRedirects as ex_req_redirects:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: TooManyRedirects".format(request_label),
errors=ex_req_redirects,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_REDIRECTS
)
except requests.exceptions.RetryError as ex_req_adapter_retry:
# Expected structure of RetryError:
# RetryError has list 'args' of len 1: RetryError.args[0] == MaxRetryError (from urlib3)
# Structure of MaxRetryError: MaxRetryError.reason == ResponseError (from urlib3)
if len(ex_req_adapter_retry.args) == 1 and hasattr(ex_req_adapter_retry.args[0], 'reason'):
response_err = ex_req_adapter_retry.args[0].reason
status_codes = [int(s) for s in response_err.args[0].split() if s.isdigit()]
if len(status_codes) == 1:
http_status_code = status_codes[0]
http_status_type = get_http_status_type(http_status_code)
error_kwargs = {
'errors': ex_req_adapter_retry,
'error_code': http_status_code,
'error_reason': response_err.args[0],
'error_message': "Request: Exception: Requests: RetryError: Exhausted: '{0}'".format(request_url),
'error_request_curl': self.built_request_curl,
}
self.logger.error(
"{0}: Exception: Requests: RetryError: Max".format(request_label),
extra=error_kwargs
)
if http_status_type == HttpStatusType.CLIENT_ERROR:
raise RequestsFortifiedClientError(**error_kwargs)
elif http_status_type == HttpStatusType.SERVER_ERROR:
raise RequestsFortifiedServiceError(**error_kwargs)
# THIS BLOCK SHOULD NOT BE ACTUALLY ACCESSED. IF IT DOES LOOK INTO IT:
self.logger.error(
"{0}: Requests RetryError: Unexpected".format(request_label),
extra={
'request_curl': self.built_request_curl
}
)
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: RetryError: Unexpected".format(request_label),
errors=ex_req_adapter_retry,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_RETRY_EXHAUSTED,
)
except requests.exceptions.RequestException as ex_req_request:
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Requests: RequestException".format(request_label),
errors=ex_req_request,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST
)
except RequestsFortifiedBaseError:
raise
except Exception as ex:
print_traceback(ex)
raise RequestsFortifiedModuleError(
error_message="{0}: Exception: Unexpected: {1}".format(request_label, base_class_name(ex)),
errors=ex,
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_SOFTWARE
)
time_end_req = dt.datetime.now()
diff_req = time_end_req - time_start_req
request_time_msecs = int(diff_req.total_seconds() * 1000)
self.logger.info(
"{0}: Finished".format(request_label),
extra={
'request_time_msecs': request_time_msecs,
}
)
self.logger.debug("{0}: Usage".format(request_label), extra=env_usage())
return response
def build_response_error_details(request_label, request_url, response):
"""Build gather status of Requests' response.
Args:
request_label:
request_url:
response:
Returns:
"""
http_status_code = \
response.status_code
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_status = "%s: %s: %s" % (http_status_code, http_status_type, http_status_desc)
response_error_details = {
'request_url': request_url,
'request_label': request_label,
'response_status': response_status,
'response_status_code': http_status_code,
'response_status_type': http_status_type,
'response_status_desc': http_status_desc
}
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_error_details.update({'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers and \
response.headers['Content-Length']:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_error_details.update({'Content-Length': response_headers_content_length})
if 'Transfer-Encoding' in response.headers and \
response.headers['Transfer-Encoding']:
response_headers_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_error_details.update({'Transfer-Encoding': response_headers_transfer_encoding})
if 'Content-Encoding' in response.headers and \
response.headers['Content-Encoding']:
response_headers_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_error_details.update({'Content-Encoding': response_headers_content_encoding})
if hasattr(response, "reason") and response.reason:
response_error_details.update({'response_reason': response.reason})
response_details = None
response_details_source = None
try:
response_details = response.json()
response_details_source = 'json'
except Exception:
if hasattr(response, 'text') and \
response.text and \
len(response.text) > 0:
response_details = response.text
response_details_source = 'text'
if response_details.startswith('<html'):
response_details_source = 'html'
soup_html = bs4.BeautifulSoup(response_details, "html.parser")
# kill all script and style elements
for script in soup_html(["script", "style"]):
script.extract() # rip it out
text_html = soup_html.get_text()
lines_html = [line for line in text_html.split('\n') if line.strip() != '']
lines_html = [line.strip(' ') for line in lines_html]
response_details = lines_html
elif response_details.startswith('<?xml'):
response_details_source = 'xml'
response_details = json.dumps(xmltodict.parse(response_details))
response_error_details.update({
'response_details': response_details,
'response_details_source': response_details_source
})
# pprint(response_error_details)
return response_error_details
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
request_cert=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
request_cert: (optional) Cert tuple to enable Client side certificates.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise TuneRequestValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise TuneRequestValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
f'{request_label}: Session: Details',
extra={
'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label
}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug(f'{request_label}: Details', extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if request_cert:
kwargs.update({'cert': request_cert})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.note(
f'{request_label}: Curl',
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug(
f'{request_label}: {request_method}',
extra={'response_url': response.url}
)
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.tune_request.request(**kwargs)
except Exception as ex:
self.logger.error(
f'{request_label}: Request Base: Error',
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
f'{request_label}: Response: Failed',
extra={
'request_curl': self.built_request_curl
},
)
raise TuneRequestModuleError(
error_message=f'{request_label}: Response: Failed',
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
f'{request_label}: Response: Details',
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
f'{request_label}: Cookie Payload',
extra={
'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label
}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error(f'{request_label}: Response: Failed', extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error(f'{request_label}: Error: Response: Details', extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error(f'{request_label}: Error: Unhandled', extra=extra_unhandled)
raise TuneRequestModuleError(**kwargs)
def request(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_retry=None,
request_retry_excps=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_cert=None,
cookie_payload=None,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False,
request_label=None
):
"""Request data from remote source with retries.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_retry: (optional) Retry configuration.
request_retry_func: (optional) Retry function, alternative
to request_retry_excps.
request_retry_excps: An exception or a tuple of exceptions
to catch.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
request_label:
Returns:
requests.Response: Data result if success or None if error.
Raises:
ServiceGatewayTimeoutError: Upon any timeout condition.
Exception: Upon error within this request_method.
Notes:
* tries: the maximum number of attempts. default: 1.
* delay: initial delay between attempts. default: 1.
* max_delay: the maximum value of delay. default: None (no limit).
* backoff: multiplier applied to delay between attempts.
default: 1 (no backoff).
* jitter: extra seconds added to delay between attempts.
default: 0.
"""
if request_label is None:
request_label = 'Request'
self.logger.debug(f'{request_label}: Start')
timeout = None
if not verify:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
if request_method:
request_method = request_method.upper()
if request_label is None:
request_label = 'Request'
if not request_retry:
request_retry = {}
if 'timeout' not in request_retry:
request_retry['timeout'] = self._REQUEST_CONFIG['timeout']
if 'tries' not in request_retry:
request_retry['tries'] = self._REQUEST_CONFIG['tries']
if 'delay' not in request_retry:
request_retry['delay'] = self._REQUEST_CONFIG['delay']
self._prep_request_retry(request_retry, request_retry_http_status_codes)
if not self.tune_request:
self.tune_request = TuneRequest(
retry_tries=self.retry_tries,
retry_backoff=self.retry_backoff,
retry_codes=self.request_retry_http_status_codes
)
key_user_agent = 'User-Agent'
header_user_agent = {key_user_agent: __USER_AGENT__}
if request_headers:
if key_user_agent not in request_headers:
request_headers.update(header_user_agent)
else:
request_headers = header_user_agent
kwargs = {
'request_method': request_method,
'request_url': request_url,
'request_params': request_params,
'request_data': request_data,
'request_json': request_json,
'request_headers': request_headers,
'request_auth': request_auth,
'request_cert': request_cert,
'cookie_payload': cookie_payload,
'request_label': request_label,
'timeout': timeout,
'build_request_curl': build_request_curl,
'allow_redirects': allow_redirects,
'verify': verify,
'stream': stream
}
time_start_req = dt.datetime.now()
if request_retry_func is not None:
self.request_retry_func = request_retry_func
if request_retry_excps_func is not None:
self.request_retry_excps_func = request_retry_excps_func
if request_retry_http_status_codes is not None:
self.request_retry_http_status_codes = request_retry_http_status_codes
if request_retry_excps is not None:
self.request_retry_excps = request_retry_excps
extra_request = copy.copy(kwargs)
if request_label:
extra_request.update({'request_label': request_label})
if request_retry:
extra_request.update({'request_retry': request_retry})
if request_retry_func:
extra_request.update({'request_retry_func': request_retry_func})
if request_retry_http_status_codes:
extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes})
if request_retry_excps:
extra_request.update({'request_retry_excps': request_retry_excps})
if request_retry_excps:
extra_request.update({'request_retry_excps_func': request_retry_excps_func})
extra_request.update(env_usage())
self.logger.debug(f'{request_label}: Start: Details', extra=extra_request)
try:
self._prep_request_retry(request_retry, request_retry_http_status_codes)
response = self._request_retry(
call_func=self._request_data,
fargs=None,
fkwargs=kwargs,
timeout=timeout,
request_label=request_label,
request_retry_func=request_retry_func,
request_retry_excps_func=request_retry_excps_func
)
except (
requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
) as ex_req_timeout:
raise TuneRequestServiceError(
error_message=f'{request_label}: Exception: Timeout',
errors=ex_req_timeout,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.HTTPError as ex_req_http:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: HTTPError',
errors=ex_req_http,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_HTTP
)
except (
requests.exceptions.ConnectionError,
) as ex_req_connect:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: ConnectionError',
errors=ex_req_connect,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (
requests.exceptions.ProxyError,
) as ex_req_proxy:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: ProxyError',
errors=ex_req_proxy,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (
requests.exceptions.SSLError,
) as ex_req_ssl:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: SSLError',
errors=ex_req_ssl,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (BrokenPipeError) as ex_broken_pipe:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: BrokenPipeError',
errors=ex_broken_pipe,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_CONNECT
)
except (ConnectionError) as ex_connect:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: ConnectionError',
errors=ex_connect,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_CONNECT
)
except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: ProtocolError',
errors=ex_req_urllib3_protocol,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout:
raise TuneRequestServiceError(
error_message=f'{request_label}: Exception: Requests: ReadTimeoutError',
errors=ex_req_urllib3_read_timeout,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.TooManyRedirects as ex_req_redirects:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: TooManyRedirects',
errors=ex_req_redirects,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_REDIRECTS
)
except requests.exceptions.RetryError as ex_req_adapter_retry:
# Expected structure of RetryError:
# RetryError has list 'args' of len 1: RetryError.args[0] == MaxRetryError (from urlib3)
# Structure of MaxRetryError: MaxRetryError.reason == ResponseError (from urlib3)
if len(ex_req_adapter_retry.args) == 1 and hasattr(ex_req_adapter_retry.args[0], 'reason'):
response_err = ex_req_adapter_retry.args[0].reason
status_codes = [int(s) for s in response_err.args[0].split() if s.isdigit()]
if len(status_codes) == 1:
http_status_code = status_codes[0]
http_status_type = get_http_status_type(http_status_code)
error_kwargs = {
'errors': ex_req_adapter_retry,
'error_code': http_status_code,
'error_reason': response_err.args[0],
'error_message': f"Request: Exception: Requests: RetryError: Exhausted: '{request_url}'",
'error_request_curl': self.built_request_curl,
}
self.logger.error(
f'{request_label}: Exception: Requests: RetryError: Max',
extra=error_kwargs
)
if http_status_type == HttpStatusType.CLIENT_ERROR:
raise TuneRequestClientError(**error_kwargs)
elif http_status_type == HttpStatusType.SERVER_ERROR:
raise TuneRequestServiceError(**error_kwargs)
# THIS BLOCK SHOULD NOT BE ACTUALLY ACCESSED. IF IT DOES LOOK INTO IT:
self.logger.error(
f'{request_label}: Requests RetryError: Unexpected',
extra={
'request_curl': self.built_request_curl
}
)
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: RetryError: Unexpected',
errors=ex_req_adapter_retry,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED,
)
except requests.exceptions.RequestException as ex_req_request:
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Requests: RequestException',
errors=ex_req_request,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST
)
except TuneRequestBaseError:
raise
except Exception as ex:
print_traceback(ex)
raise TuneRequestModuleError(
error_message=f'{request_label}: Exception: Unexpected: {base_class_name(ex)}',
errors=ex,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE
)
time_end_req = dt.datetime.now()
diff_req = time_end_req - time_start_req
request_time_msecs = int(diff_req.total_seconds() * 1000)
self.logger.info(
f'{request_label}: Finished',
extra={
'request_time_msecs': request_time_msecs,
}
)
self.logger.debug(f'{request_label}: Usage', extra=env_usage())
return response
def build_response_error_details(request_label, request_url, response):
"""Build gather status of Requests' response.
Args:
request_label:
request_url:
response:
Returns:
"""
http_status_code = \
response.status_code
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_status = f"{http_status_code}: {http_status_type}: {http_status_desc}"
response_error_details = {
'request_url': request_url,
'request_label': request_label,
'response_status': response_status,
'response_status_code': http_status_code,
'response_status_type': http_status_type,
'response_status_desc': http_status_desc
}
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_error_details.update(
{'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers and \
response.headers['Content-Length']:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_error_details.update(
{'Content-Length': response_headers_content_length})
if 'Transfer-Encoding' in response.headers and \
response.headers['Transfer-Encoding']:
response_headers_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_error_details.update(
{'Transfer-Encoding': response_headers_transfer_encoding})
if 'Content-Encoding' in response.headers and \
response.headers['Content-Encoding']:
response_headers_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_error_details.update(
{'Content-Encoding': response_headers_content_encoding})
if hasattr(response, "reason") and response.reason:
response_error_details.update({'response_reason': response.reason})
response_details = None
response_details_source = None
try:
response_details = response.json()
response_details_source = 'json'
except Exception:
if hasattr(response, 'text') and \
response.text and \
len(response.text) > 0:
response_details = response.text
response_details_source = 'text'
if response_details.startswith('<html'):
response_details_source = 'html'
soup_html = bs4.BeautifulSoup(response_details, "html.parser")
# kill all script and style elements
for script in soup_html(["script", "style"]):
script.extract() # rip it out
text_html = soup_html.get_text()
lines_html = [
line for line in text_html.split('\n')
if line.strip() != ''
]
lines_html = [line.strip(' ') for line in lines_html]
response_details = lines_html
elif response_details.startswith('<?xml'):
response_details_source = 'xml'
response_details = json.dumps(
xmltodict.parse(response_details))
response_error_details.update({
'response_details':
response_details,
'response_details_source':
response_details_source
})
# pprint(response_error_details)
return response_error_details
