def test_http_status_code_to_type(self): __all_http_status_codes__ = list(map(int, HTTPStatus)) for http_status_code in __all_http_status_codes__: assert(pyhttpstatus_utils.get_http_status_type(http_status_code)) assert(type(pyhttpstatus_utils.is_http_status_successful(http_status_code)) is bool) if pyhttpstatus_utils.is_http_status_successful(http_status_code): assert(pyhttpstatus_utils.get_http_status_type( http_status_code) == pyhttpstatus_utils.HttpStatusType.SUCCESSFUL) else: assert(pyhttpstatus_utils.get_http_status_type( http_status_code) != pyhttpstatus_utils.HttpStatusType.SUCCESSFUL)
def get_http_responses_4xx_5xx(): client_error_type = get_http_status_type(400) server_error_type = get_http_status_type(500) ignore_list = [419, 425, 509] # Uncommon http responses excluded from the test # Collect all possible 4xx and 5xx http responses http_responses = [ code for code in HTTP_STATUS_PHRASE_DICT if get_http_status_type(code) in [client_error_type, server_error_type] ] http_responses = [ code for code in http_responses if code not in ignore_list ] return http_responses
def tune_v2_request_retry_func(self, response): """Request Retry Function Args: response: Returns: Boolean """ try: response_json = response.json() except Exception as ex: # No JSON response available. raise TuneReportingError( error_message='Invalid JSON response: {}'.format(response.text), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_JSON_DECODING ) self.logger.debug("TMC API V2: Check for Retry: Start", extra=response_json) tune_v2_status_code = None tune_v2_errors_messages = "" if 'status_code' in response_json: tune_v2_status_code = response_json['status_code'] if 'errors' in response_json: tune_v2_errors = response_json['errors'] if isinstance(tune_v2_errors, dict) \ and 'message' in tune_v2_errors: tune_v2_errors_messages += tune_v2_errors['message'] elif isinstance(tune_v2_errors, list): for tune_v2_error in tune_v2_errors: if isinstance(tune_v2_error, dict) \ and 'message' in tune_v2_error: tune_v2_errors_messages += tune_v2_error['message'] tune_v2_status_type = get_http_status_type(tune_v2_status_code) response_extra = { 'status_code': tune_v2_status_code, 'status_type': tune_v2_status_type, } if tune_v2_errors_messages: response_extra.update({'error_messages': safe_str(tune_v2_errors_messages)}) self.logger.debug("TMC API Base: Check for Retry: Response", extra=response_extra) if tune_v2_status_code == 200: self.logger.debug("TMC API Base: Check for Retry: Success", extra=response_extra) return False if tune_v2_status_code in [401, 403]: self.logger.error("TMC API: Request: Error", extra={'status_code': tune_v2_status_code}) raise TuneRequestError( error_message=tune_v2_errors_messages, error_code=tune_v2_status_code, ) if tune_v2_status_code in [404, 500]: if "Api key was not found." in tune_v2_errors_messages: self.logger.error("TMC API: Request: Error", extra={'tune_v2_status_code': tune_v2_status_code}) raise TuneRequestError( error_message=tune_v2_errors_messages, error_code=tune_v2_status_code, ) self.logger.warning("TMC API Base: Check for Retry: Retry Candidate", extra=response_extra) return True self.logger.warning("TMC API Base: Check for Retry: No Retry", extra=response_extra) return False
def _request_data( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_headers=None, request_auth=None, request_cert=None, cookie_payload=None, request_label=None, timeout=60, build_request_curl=True, allow_redirects=True, verify=True, stream=False ): """Request Data from requests. Args: request_method: request_method for the new :class:`Request` object. logger: logging instance request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. request_cert: (optional) Cert tuple to enable Client side certificates. timeout: (optional) How long to wait for the server to send data before giving up. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. Returns: requests.Response """ if request_label is None: request_label = 'Request Data' if not request_method: raise RequestsFortifiedValueError(error_message="Parameter 'request_method' not defined") if not request_url: raise RequestsFortifiedValueError(error_message="Parameter 'request_url' not defined") self.built_request_curl = None self.logger.debug( "{0}: Session: Details".format(request_label), extra={ 'cookie_payload': self.requests_session_client.session.cookies.get_dict(), 'request_label': request_label } ) response = None headers = None if request_headers: headers = request_headers request_method = request_method.upper() if request_data and isinstance(request_data, str): if len(request_data) <= 20: request_data_extra = request_data else: request_data_extra = request_data[:20] + ' ...' else: request_data_extra = safe_str(request_data) request_extra = { 'request_method': request_method, 'request_url': request_url, 'timeout': timeout, 'request_params': safe_dict(request_params), 'request_data': request_data_extra, 'request_headers': safe_dict(headers), 'request_label': request_label } self.logger.debug("{0}: Details".format(request_label), extra=request_extra) self.built_request_curl = None kwargs = {} if headers: kwargs.update({'headers': headers}) if request_auth: kwargs.update({'auth': request_auth}) if request_cert: kwargs.update({'cert': request_cert}) if timeout and isinstance(timeout, int): kwargs.update({'timeout': timeout}) if allow_redirects: kwargs.update({'allow_redirects': allow_redirects}) if stream: kwargs.update({'stream': stream}) if cookie_payload: kwargs.update({'cookies': cookie_payload}) kwargs.update({'verify': verify}) try: if build_request_curl: # In case no authentication information has been provided, # use session's cookies information, if exists if not request_auth and self.session.cookies and len(self.session.cookies) > 0: request_auth = self.session.cookies self.built_request_curl = command_line_request_curl( request_method=request_method, request_url=request_url, request_headers=headers, request_params=request_params, request_data=request_data, request_json=request_json, request_auth=request_auth, request_timeout=timeout, request_allow_redirects=allow_redirects ) self.logger.note( "{0}: Curl".format(request_label), extra={ 'request_method': request_method, 'request_label': request_label, 'request_curl': self.built_request_curl } ) if hasattr(response, 'url'): self.logger.debug( "{0}: {1}".format(request_label, request_method), extra={'response_url': response.url} ) if request_params: kwargs.update({'params': request_params}) if request_data: kwargs.update({'data': request_data}) if request_json: kwargs.update({'json': request_json}) if headers: kwargs.update({'headers': headers}) kwargs.update({'request_method': request_method, 'request_url': request_url}) response = self.requests_session_client.request(**kwargs) except Exception as ex: self.logger.error( "{0}: Request Base: Error".format(request_label), extra={ 'request_label': request_label, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) } ) raise if response is None: self.logger.error( "{0}: Response: Failed".format(request_label), extra={ 'request_curl': self.built_request_curl }, ) raise RequestsFortifiedModuleError( error_message="{0}: Response: Failed".format(request_label), error_code=RequestsFortifiedErrorCodes.REQ_ERR_UNEXPECTED_VALUE, error_request_curl=self.built_request_curl ) http_status_code = response.status_code response_headers = json.loads(json.dumps(dict(response.headers))) http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_extra = { 'http_status_code': http_status_code, 'http_status_type': http_status_type, 'http_status_desc': http_status_desc, 'response_headers': safe_dict(response_headers), } self.logger.debug( "{0}: Response: Details".format(request_label), extra=response_extra, ) http_status_successful = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL ) http_status_redirection = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION ) if http_status_successful or http_status_redirection: if hasattr(response, 'url') and \ response.url and \ len(response.url) > 0: response_extra.update({'response_url': response.url}) self.logger.debug( "{0}: Cookie Payload".format(request_label), extra={ 'cookie_payload': self.requests_session_client.session.cookies.get_dict(), 'request_label': request_label } ) assert response return response else: response_extra.update({'error_request_curl': self.built_request_curl}) self.logger.error("{0}: Response: Failed".format(request_label), extra=response_extra) json_response_error = \ build_response_error_details( response=response, request_label=request_label, request_url=request_url ) extra_error = copy.deepcopy(json_response_error) if self.logger_level == logging.INFO: error_response_details = \ extra_error.get('response_details', None) if error_response_details and \ isinstance(error_response_details, str) and \ len(error_response_details) > 100: extra_error['response_details'] = error_response_details[:100] + ' ...' if self.built_request_curl and \ 'error_request_curl' not in extra_error: extra_error.update({'error_request_curl': self.built_request_curl}) self.logger.error("{0}: Error: Response: Details".format(request_label), extra=extra_error) kwargs = { 'error_status': json_response_error.get("response_status", None), 'error_reason': json_response_error.get("response_reason", None), 'error_details': json_response_error.get("response_details", None), 'error_request_curl': self.built_request_curl } if http_status_code in [ HttpStatusCode.BAD_REQUEST, HttpStatusCode.UNAUTHORIZED, HttpStatusCode.FORBIDDEN, HttpStatusCode.NOT_FOUND, HttpStatusCode.METHOD_NOT_ALLOWED, HttpStatusCode.NOT_ACCEPTABLE, HttpStatusCode.REQUEST_TIMEOUT, HttpStatusCode.CONFLICT, HttpStatusCode.GONE, HttpStatusCode.UNPROCESSABLE_ENTITY, HttpStatusCode.TOO_MANY_REQUESTS, ]: kwargs.update({'error_code': http_status_code}) raise RequestsFortifiedClientError(**kwargs) if http_status_code in [ HttpStatusCode.INTERNAL_SERVER_ERROR, HttpStatusCode.NOT_IMPLEMENTED, HttpStatusCode.BAD_GATEWAY, HttpStatusCode.SERVICE_UNAVAILABLE, HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED, ]: kwargs.update({'error_code': http_status_code}) raise RequestsFortifiedServiceError(**kwargs) kwargs.update({'error_code': json_response_error['response_status_code']}) extra_unhandled = copy.deepcopy(kwargs) extra_unhandled.update({'http_status_code': http_status_code}) self.logger.error("{0}: Error: Unhandled".format(request_label), extra=extra_unhandled) raise RequestsFortifiedModuleError(**kwargs)
def request( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_retry=None, request_retry_excps=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_headers=None, request_auth=None, request_cert=None, cookie_payload=None, build_request_curl=True, allow_redirects=True, verify=True, stream=False, request_label=None ): """Request data from remote source with retries. Args: request_method: request_method for the new :class:`Request` object. request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_retry: (optional) Retry configuration. request_retry_func: (optional) Retry function, alternative to request_retry_excps. request_retry_excps: An exception or a tuple of exceptions to catch. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. request_label: Returns: requests.Response: Data result if success or None if error. Raises: ServiceGatewayTimeoutError: Upon any timeout condition. Exception: Upon error within this request_method. Notes: * tries: the maximum number of attempts. default: 1. * delay: initial delay between attempts. default: 1. * max_delay: the maximum value of delay. default: None (no limit). * backoff: multiplier applied to delay between attempts. default: 1 (no backoff). * jitter: extra seconds added to delay between attempts. default: 0. """ if request_label is None: request_label = 'Request' self.logger.debug( "{0}: Start".format(request_label) ) timeout = None if not verify: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) if request_method: request_method = request_method.upper() if request_label is None: request_label = 'Request' if not request_retry: request_retry = {} if 'timeout' not in request_retry: request_retry['timeout'] = self._REQUEST_CONFIG['timeout'] if 'tries' not in request_retry: request_retry['tries'] = self._REQUEST_CONFIG['tries'] if 'delay' not in request_retry: request_retry['delay'] = self._REQUEST_CONFIG['delay'] self._prep_request_retry(request_retry, request_retry_http_status_codes) if not self.requests_session_client: self.requests_session_client = RequestsSessionClient( retry_tries=self.retry_tries, retry_backoff=self.retry_backoff, retry_codes=self.request_retry_http_status_codes ) key_user_agent = 'User-Agent' header_user_agent = {key_user_agent: __USER_AGENT__} if request_headers: if key_user_agent not in request_headers: request_headers.update(header_user_agent) else: request_headers = header_user_agent kwargs = { 'request_method': request_method, 'request_url': request_url, 'request_params': request_params, 'request_data': request_data, 'request_json': request_json, 'request_headers': request_headers, 'request_auth': request_auth, 'request_cert': request_cert, 'cookie_payload': cookie_payload, 'request_label': request_label, 'timeout': timeout, 'build_request_curl': build_request_curl, 'allow_redirects': allow_redirects, 'verify': verify, 'stream': stream } time_start_req = dt.datetime.now() if request_retry_func is not None: self.request_retry_func = request_retry_func if request_retry_excps_func is not None: self.request_retry_excps_func = request_retry_excps_func if request_retry_http_status_codes is not None: self.request_retry_http_status_codes = request_retry_http_status_codes if request_retry_excps is not None: self.request_retry_excps = request_retry_excps extra_request = copy.copy(kwargs) if request_label: extra_request.update({'request_label': request_label}) if request_retry: extra_request.update({'request_retry': request_retry}) if request_retry_func: extra_request.update({'request_retry_func': request_retry_func}) if request_retry_http_status_codes: extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes}) if request_retry_excps: extra_request.update({'request_retry_excps': request_retry_excps}) if request_retry_excps: extra_request.update({'request_retry_excps_func': request_retry_excps_func}) extra_request.update(env_usage()) self.logger.debug("{0}: Start: Details".format(request_label), extra=extra_request) try: self._prep_request_retry(request_retry, request_retry_http_status_codes) response = self._request_retry( call_func=self._request_data, fargs=None, fkwargs=kwargs, timeout=timeout, request_label=request_label, request_retry_func=request_retry_func, request_retry_excps_func=request_retry_excps_func ) except ( requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout, ) as ex_req_timeout: raise RequestsFortifiedServiceError( error_message="{0}: Exception: Timeout".format(request_label), errors=ex_req_timeout, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.HTTPError as ex_req_http: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: HTTPError".format(request_label), errors=ex_req_http, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_HTTP ) except ( requests.exceptions.ConnectionError, ) as ex_req_connect: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: ConnectionError".format(request_label), errors=ex_req_connect, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT ) except ( requests.exceptions.ProxyError, ) as ex_req_proxy: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: ProxyError".format(request_label), errors=ex_req_proxy, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT ) except ( requests.exceptions.SSLError, ) as ex_req_ssl: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: SSLError".format(request_label), errors=ex_req_ssl, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT ) except (BrokenPipeError) as ex_broken_pipe: raise RequestsFortifiedModuleError( error_message="{0}: Exception: BrokenPipeError".format(request_label), errors=ex_broken_pipe, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_CONNECT ) except (ConnectionError) as ex_connect: raise RequestsFortifiedModuleError( error_message="{0}: Exception: ConnectionError".format(request_label), errors=ex_connect, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_CONNECT ) except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: ProtocolError".format(request_label), errors=ex_req_urllib3_protocol, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_CONNECT ) except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout: raise RequestsFortifiedServiceError( error_message="{0}: Exception: Requests: ReadTimeoutError".format(request_label), errors=ex_req_urllib3_read_timeout, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.TooManyRedirects as ex_req_redirects: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: TooManyRedirects".format(request_label), errors=ex_req_redirects, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST_REDIRECTS ) except requests.exceptions.RetryError as ex_req_adapter_retry: # Expected structure of RetryError: # RetryError has list 'args' of len 1: RetryError.args[0] == MaxRetryError (from urlib3) # Structure of MaxRetryError: MaxRetryError.reason == ResponseError (from urlib3) if len(ex_req_adapter_retry.args) == 1 and hasattr(ex_req_adapter_retry.args[0], 'reason'): response_err = ex_req_adapter_retry.args[0].reason status_codes = [int(s) for s in response_err.args[0].split() if s.isdigit()] if len(status_codes) == 1: http_status_code = status_codes[0] http_status_type = get_http_status_type(http_status_code) error_kwargs = { 'errors': ex_req_adapter_retry, 'error_code': http_status_code, 'error_reason': response_err.args[0], 'error_message': "Request: Exception: Requests: RetryError: Exhausted: '{0}'".format(request_url), 'error_request_curl': self.built_request_curl, } self.logger.error( "{0}: Exception: Requests: RetryError: Max".format(request_label), extra=error_kwargs ) if http_status_type == HttpStatusType.CLIENT_ERROR: raise RequestsFortifiedClientError(**error_kwargs) elif http_status_type == HttpStatusType.SERVER_ERROR: raise RequestsFortifiedServiceError(**error_kwargs) # THIS BLOCK SHOULD NOT BE ACTUALLY ACCESSED. IF IT DOES LOOK INTO IT: self.logger.error( "{0}: Requests RetryError: Unexpected".format(request_label), extra={ 'request_curl': self.built_request_curl } ) raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: RetryError: Unexpected".format(request_label), errors=ex_req_adapter_retry, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_RETRY_EXHAUSTED, ) except requests.exceptions.RequestException as ex_req_request: raise RequestsFortifiedModuleError( error_message="{0}: Exception: Requests: RequestException".format(request_label), errors=ex_req_request, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST ) except RequestsFortifiedBaseError: raise except Exception as ex: print_traceback(ex) raise RequestsFortifiedModuleError( error_message="{0}: Exception: Unexpected: {1}".format(request_label, base_class_name(ex)), errors=ex, error_request_curl=self.built_request_curl, error_code=RequestsFortifiedErrorCodes.REQ_ERR_SOFTWARE ) time_end_req = dt.datetime.now() diff_req = time_end_req - time_start_req request_time_msecs = int(diff_req.total_seconds() * 1000) self.logger.info( "{0}: Finished".format(request_label), extra={ 'request_time_msecs': request_time_msecs, } ) self.logger.debug("{0}: Usage".format(request_label), extra=env_usage()) return response
def build_response_error_details(request_label, request_url, response): """Build gather status of Requests' response. Args: request_label: request_url: response: Returns: """ http_status_code = \ response.status_code http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_status = "%s: %s: %s" % (http_status_code, http_status_type, http_status_desc) response_error_details = { 'request_url': request_url, 'request_label': request_label, 'response_status': response_status, 'response_status_code': http_status_code, 'response_status_type': http_status_type, 'response_status_desc': http_status_desc } if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_error_details.update({'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers and \ response.headers['Content-Length']: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_error_details.update({'Content-Length': response_headers_content_length}) if 'Transfer-Encoding' in response.headers and \ response.headers['Transfer-Encoding']: response_headers_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_error_details.update({'Transfer-Encoding': response_headers_transfer_encoding}) if 'Content-Encoding' in response.headers and \ response.headers['Content-Encoding']: response_headers_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_error_details.update({'Content-Encoding': response_headers_content_encoding}) if hasattr(response, "reason") and response.reason: response_error_details.update({'response_reason': response.reason}) response_details = None response_details_source = None try: response_details = response.json() response_details_source = 'json' except Exception: if hasattr(response, 'text') and \ response.text and \ len(response.text) > 0: response_details = response.text response_details_source = 'text' if response_details.startswith('<html'): response_details_source = 'html' soup_html = bs4.BeautifulSoup(response_details, "html.parser") # kill all script and style elements for script in soup_html(["script", "style"]): script.extract() # rip it out text_html = soup_html.get_text() lines_html = [line for line in text_html.split('\n') if line.strip() != ''] lines_html = [line.strip(' ') for line in lines_html] response_details = lines_html elif response_details.startswith('<?xml'): response_details_source = 'xml' response_details = json.dumps(xmltodict.parse(response_details)) response_error_details.update({ 'response_details': response_details, 'response_details_source': response_details_source }) # pprint(response_error_details) return response_error_details
def _request_data( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_headers=None, request_auth=None, request_cert=None, cookie_payload=None, request_label=None, timeout=60, build_request_curl=True, allow_redirects=True, verify=True, stream=False ): """Request Data from requests. Args: request_method: request_method for the new :class:`Request` object. logger: logging instance request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. request_cert: (optional) Cert tuple to enable Client side certificates. timeout: (optional) How long to wait for the server to send data before giving up. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. Returns: requests.Response """ if request_label is None: request_label = 'Request Data' if not request_method: raise TuneRequestValueError(error_message="Parameter 'request_method' not defined") if not request_url: raise TuneRequestValueError(error_message="Parameter 'request_url' not defined") self.built_request_curl = None self.logger.debug( f'{request_label}: Session: Details', extra={ 'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label } ) response = None headers = None if request_headers: headers = request_headers request_method = request_method.upper() if request_data and isinstance(request_data, str): if len(request_data) <= 20: request_data_extra = request_data else: request_data_extra = request_data[:20] + ' ...' else: request_data_extra = safe_str(request_data) request_extra = { 'request_method': request_method, 'request_url': request_url, 'timeout': timeout, 'request_params': safe_dict(request_params), 'request_data': request_data_extra, 'request_headers': safe_dict(headers), 'request_label': request_label } self.logger.debug(f'{request_label}: Details', extra=request_extra) self.built_request_curl = None kwargs = {} if headers: kwargs.update({'headers': headers}) if request_auth: kwargs.update({'auth': request_auth}) if request_cert: kwargs.update({'cert': request_cert}) if timeout and isinstance(timeout, int): kwargs.update({'timeout': timeout}) if allow_redirects: kwargs.update({'allow_redirects': allow_redirects}) if stream: kwargs.update({'stream': stream}) if cookie_payload: kwargs.update({'cookies': cookie_payload}) kwargs.update({'verify': verify}) try: if build_request_curl: # In case no authentication information has been provided, # use session's cookies information, if exists if not request_auth and self.session.cookies and len(self.session.cookies) > 0: request_auth = self.session.cookies self.built_request_curl = command_line_request_curl( request_method=request_method, request_url=request_url, request_headers=headers, request_params=request_params, request_data=request_data, request_json=request_json, request_auth=request_auth, request_timeout=timeout, request_allow_redirects=allow_redirects ) self.logger.note( f'{request_label}: Curl', extra={ 'request_method': request_method, 'request_label': request_label, 'request_curl': self.built_request_curl } ) if hasattr(response, 'url'): self.logger.debug( f'{request_label}: {request_method}', extra={'response_url': response.url} ) if request_params: kwargs.update({'params': request_params}) if request_data: kwargs.update({'data': request_data}) if request_json: kwargs.update({'json': request_json}) if headers: kwargs.update({'headers': headers}) kwargs.update({'request_method': request_method, 'request_url': request_url}) response = self.tune_request.request(**kwargs) except Exception as ex: self.logger.error( f'{request_label}: Request Base: Error', extra={ 'request_label': request_label, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) } ) raise if response is None: self.logger.error( f'{request_label}: Response: Failed', extra={ 'request_curl': self.built_request_curl }, ) raise TuneRequestModuleError( error_message=f'{request_label}: Response: Failed', error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE, error_request_curl=self.built_request_curl ) http_status_code = response.status_code response_headers = json.loads(json.dumps(dict(response.headers))) http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_extra = { 'http_status_code': http_status_code, 'http_status_type': http_status_type, 'http_status_desc': http_status_desc, 'response_headers': safe_dict(response_headers), } self.logger.debug( f'{request_label}: Response: Details', extra=response_extra, ) http_status_successful = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL ) http_status_redirection = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION ) if http_status_successful or http_status_redirection: if hasattr(response, 'url') and \ response.url and \ len(response.url) > 0: response_extra.update({'response_url': response.url}) self.logger.debug( f'{request_label}: Cookie Payload', extra={ 'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label } ) assert response return response else: response_extra.update({'error_request_curl': self.built_request_curl}) self.logger.error(f'{request_label}: Response: Failed', extra=response_extra) json_response_error = \ build_response_error_details( response=response, request_label=request_label, request_url=request_url ) extra_error = copy.deepcopy(json_response_error) if self.logger_level == logging.INFO: error_response_details = \ extra_error.get('response_details', None) if error_response_details and \ isinstance(error_response_details, str) and \ len(error_response_details) > 100: extra_error['response_details'] = error_response_details[:100] + ' ...' if self.built_request_curl and \ 'error_request_curl' not in extra_error: extra_error.update({'error_request_curl': self.built_request_curl}) self.logger.error(f'{request_label}: Error: Response: Details', extra=extra_error) kwargs = { 'error_status': json_response_error.get("response_status", None), 'error_reason': json_response_error.get("response_reason", None), 'error_details': json_response_error.get("response_details", None), 'error_request_curl': self.built_request_curl } if http_status_code in [ HttpStatusCode.BAD_REQUEST, HttpStatusCode.UNAUTHORIZED, HttpStatusCode.FORBIDDEN, HttpStatusCode.NOT_FOUND, HttpStatusCode.METHOD_NOT_ALLOWED, HttpStatusCode.NOT_ACCEPTABLE, HttpStatusCode.REQUEST_TIMEOUT, HttpStatusCode.CONFLICT, HttpStatusCode.GONE, HttpStatusCode.UNPROCESSABLE_ENTITY, HttpStatusCode.TOO_MANY_REQUESTS, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestClientError(**kwargs) if http_status_code in [ HttpStatusCode.INTERNAL_SERVER_ERROR, HttpStatusCode.NOT_IMPLEMENTED, HttpStatusCode.BAD_GATEWAY, HttpStatusCode.SERVICE_UNAVAILABLE, HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestServiceError(**kwargs) kwargs.update({'error_code': json_response_error['response_status_code']}) extra_unhandled = copy.deepcopy(kwargs) extra_unhandled.update({'http_status_code': http_status_code}) self.logger.error(f'{request_label}: Error: Unhandled', extra=extra_unhandled) raise TuneRequestModuleError(**kwargs)
def request( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_retry=None, request_retry_excps=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_headers=None, request_auth=None, request_cert=None, cookie_payload=None, build_request_curl=True, allow_redirects=True, verify=True, stream=False, request_label=None ): """Request data from remote source with retries. Args: request_method: request_method for the new :class:`Request` object. request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_retry: (optional) Retry configuration. request_retry_func: (optional) Retry function, alternative to request_retry_excps. request_retry_excps: An exception or a tuple of exceptions to catch. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. request_label: Returns: requests.Response: Data result if success or None if error. Raises: ServiceGatewayTimeoutError: Upon any timeout condition. Exception: Upon error within this request_method. Notes: * tries: the maximum number of attempts. default: 1. * delay: initial delay between attempts. default: 1. * max_delay: the maximum value of delay. default: None (no limit). * backoff: multiplier applied to delay between attempts. default: 1 (no backoff). * jitter: extra seconds added to delay between attempts. default: 0. """ if request_label is None: request_label = 'Request' self.logger.debug(f'{request_label}: Start') timeout = None if not verify: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) if request_method: request_method = request_method.upper() if request_label is None: request_label = 'Request' if not request_retry: request_retry = {} if 'timeout' not in request_retry: request_retry['timeout'] = self._REQUEST_CONFIG['timeout'] if 'tries' not in request_retry: request_retry['tries'] = self._REQUEST_CONFIG['tries'] if 'delay' not in request_retry: request_retry['delay'] = self._REQUEST_CONFIG['delay'] self._prep_request_retry(request_retry, request_retry_http_status_codes) if not self.tune_request: self.tune_request = TuneRequest( retry_tries=self.retry_tries, retry_backoff=self.retry_backoff, retry_codes=self.request_retry_http_status_codes ) key_user_agent = 'User-Agent' header_user_agent = {key_user_agent: __USER_AGENT__} if request_headers: if key_user_agent not in request_headers: request_headers.update(header_user_agent) else: request_headers = header_user_agent kwargs = { 'request_method': request_method, 'request_url': request_url, 'request_params': request_params, 'request_data': request_data, 'request_json': request_json, 'request_headers': request_headers, 'request_auth': request_auth, 'request_cert': request_cert, 'cookie_payload': cookie_payload, 'request_label': request_label, 'timeout': timeout, 'build_request_curl': build_request_curl, 'allow_redirects': allow_redirects, 'verify': verify, 'stream': stream } time_start_req = dt.datetime.now() if request_retry_func is not None: self.request_retry_func = request_retry_func if request_retry_excps_func is not None: self.request_retry_excps_func = request_retry_excps_func if request_retry_http_status_codes is not None: self.request_retry_http_status_codes = request_retry_http_status_codes if request_retry_excps is not None: self.request_retry_excps = request_retry_excps extra_request = copy.copy(kwargs) if request_label: extra_request.update({'request_label': request_label}) if request_retry: extra_request.update({'request_retry': request_retry}) if request_retry_func: extra_request.update({'request_retry_func': request_retry_func}) if request_retry_http_status_codes: extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes}) if request_retry_excps: extra_request.update({'request_retry_excps': request_retry_excps}) if request_retry_excps: extra_request.update({'request_retry_excps_func': request_retry_excps_func}) extra_request.update(env_usage()) self.logger.debug(f'{request_label}: Start: Details', extra=extra_request) try: self._prep_request_retry(request_retry, request_retry_http_status_codes) response = self._request_retry( call_func=self._request_data, fargs=None, fkwargs=kwargs, timeout=timeout, request_label=request_label, request_retry_func=request_retry_func, request_retry_excps_func=request_retry_excps_func ) except ( requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout, ) as ex_req_timeout: raise TuneRequestServiceError( error_message=f'{request_label}: Exception: Timeout', errors=ex_req_timeout, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.HTTPError as ex_req_http: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: HTTPError', errors=ex_req_http, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_HTTP ) except ( requests.exceptions.ConnectionError, ) as ex_req_connect: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: ConnectionError', errors=ex_req_connect, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except ( requests.exceptions.ProxyError, ) as ex_req_proxy: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: ProxyError', errors=ex_req_proxy, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except ( requests.exceptions.SSLError, ) as ex_req_ssl: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: SSLError', errors=ex_req_ssl, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except (BrokenPipeError) as ex_broken_pipe: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: BrokenPipeError', errors=ex_broken_pipe, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_CONNECT ) except (ConnectionError) as ex_connect: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: ConnectionError', errors=ex_connect, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_CONNECT ) except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: ProtocolError', errors=ex_req_urllib3_protocol, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout: raise TuneRequestServiceError( error_message=f'{request_label}: Exception: Requests: ReadTimeoutError', errors=ex_req_urllib3_read_timeout, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.TooManyRedirects as ex_req_redirects: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: TooManyRedirects', errors=ex_req_redirects, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_REDIRECTS ) except requests.exceptions.RetryError as ex_req_adapter_retry: # Expected structure of RetryError: # RetryError has list 'args' of len 1: RetryError.args[0] == MaxRetryError (from urlib3) # Structure of MaxRetryError: MaxRetryError.reason == ResponseError (from urlib3) if len(ex_req_adapter_retry.args) == 1 and hasattr(ex_req_adapter_retry.args[0], 'reason'): response_err = ex_req_adapter_retry.args[0].reason status_codes = [int(s) for s in response_err.args[0].split() if s.isdigit()] if len(status_codes) == 1: http_status_code = status_codes[0] http_status_type = get_http_status_type(http_status_code) error_kwargs = { 'errors': ex_req_adapter_retry, 'error_code': http_status_code, 'error_reason': response_err.args[0], 'error_message': f"Request: Exception: Requests: RetryError: Exhausted: '{request_url}'", 'error_request_curl': self.built_request_curl, } self.logger.error( f'{request_label}: Exception: Requests: RetryError: Max', extra=error_kwargs ) if http_status_type == HttpStatusType.CLIENT_ERROR: raise TuneRequestClientError(**error_kwargs) elif http_status_type == HttpStatusType.SERVER_ERROR: raise TuneRequestServiceError(**error_kwargs) # THIS BLOCK SHOULD NOT BE ACTUALLY ACCESSED. IF IT DOES LOOK INTO IT: self.logger.error( f'{request_label}: Requests RetryError: Unexpected', extra={ 'request_curl': self.built_request_curl } ) raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: RetryError: Unexpected', errors=ex_req_adapter_retry, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED, ) except requests.exceptions.RequestException as ex_req_request: raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Requests: RequestException', errors=ex_req_request, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST ) except TuneRequestBaseError: raise except Exception as ex: print_traceback(ex) raise TuneRequestModuleError( error_message=f'{request_label}: Exception: Unexpected: {base_class_name(ex)}', errors=ex, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE ) time_end_req = dt.datetime.now() diff_req = time_end_req - time_start_req request_time_msecs = int(diff_req.total_seconds() * 1000) self.logger.info( f'{request_label}: Finished', extra={ 'request_time_msecs': request_time_msecs, } ) self.logger.debug(f'{request_label}: Usage', extra=env_usage()) return response
def build_response_error_details(request_label, request_url, response): """Build gather status of Requests' response. Args: request_label: request_url: response: Returns: """ http_status_code = \ response.status_code http_status_type = \ get_http_status_type(http_status_code) http_status_desc = \ get_http_status_desc(http_status_code) response_status = f"{http_status_code}: {http_status_type}: {http_status_desc}" response_error_details = { 'request_url': request_url, 'request_label': request_label, 'response_status': response_status, 'response_status_code': http_status_code, 'response_status_type': http_status_type, 'response_status_desc': http_status_desc } if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_error_details.update( {'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers and \ response.headers['Content-Length']: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_error_details.update( {'Content-Length': response_headers_content_length}) if 'Transfer-Encoding' in response.headers and \ response.headers['Transfer-Encoding']: response_headers_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_error_details.update( {'Transfer-Encoding': response_headers_transfer_encoding}) if 'Content-Encoding' in response.headers and \ response.headers['Content-Encoding']: response_headers_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_error_details.update( {'Content-Encoding': response_headers_content_encoding}) if hasattr(response, "reason") and response.reason: response_error_details.update({'response_reason': response.reason}) response_details = None response_details_source = None try: response_details = response.json() response_details_source = 'json' except Exception: if hasattr(response, 'text') and \ response.text and \ len(response.text) > 0: response_details = response.text response_details_source = 'text' if response_details.startswith('<html'): response_details_source = 'html' soup_html = bs4.BeautifulSoup(response_details, "html.parser") # kill all script and style elements for script in soup_html(["script", "style"]): script.extract() # rip it out text_html = soup_html.get_text() lines_html = [ line for line in text_html.split('\n') if line.strip() != '' ] lines_html = [line.strip(' ') for line in lines_html] response_details = lines_html elif response_details.startswith('<?xml'): response_details_source = 'xml' response_details = json.dumps( xmltodict.parse(response_details)) response_error_details.update({ 'response_details': response_details, 'response_details_source': response_details_source }) # pprint(response_error_details) return response_error_details