def try_send_request(self, _attempts, _tries, request_func, request_label, request_retry_func, request_url): to_raise_exception = None to_return_response = None try: response = request_func() if response is None: raise TuneRequestModuleError( error_message="Request Retry: No response", error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE ) if self.is_return_response(request_label, request_retry_func, request_url, response): to_return_response = response else: self.logger.debug( "Request Retry: Response: Valid: Retry Candidate", extra={'request_url': request_url, 'request_label': request_label} ) except tuple(self.request_retry_excps) as retry_ex: if not self.is_retry_retry_ex(_tries, request_label, request_url, retry_ex): to_raise_exception = retry_ex except TuneRequestBaseError as tmv_ex: if not self.is_retry_non_retry_ex(_tries, request_label, tmv_ex): to_raise_exception = tmv_ex except Exception as ex: is_retry, raised_exception = self.is_retry_non_tune_ex(_tries, ex, request_label, request_url) if not is_retry: to_raise_exception = raised_exception # A final check, whether we need to raise an exception, is in case the number of retries has exhausted. if not to_raise_exception and self.is_exhausted_retries( _tries, partial( self.logger.error, "Request Retry: Exhausted Retries", extra={ 'attempts': _attempts, 'tries': _tries, 'request_url': request_url, 'request_label': request_label } ) ): to_raise_exception = TuneRequestModuleError( error_message=("Request Retry: Exhausted Retries: {}: {}").format(request_label, request_url), error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED ) return to_raise_exception, to_return_response
def exceptions(): exceptions_dict = dict() exceptions_dict[RequestRetryException.__name__] = RequestRetryException() exceptions_dict[TuneRequestBaseError.__name__] = TuneRequestBaseError() exceptions_dict[Exception.__name__] = Exception() exceptions_dict[TuneRequestModuleError.__name__] = dict() exceptions_dict[TuneRequestModuleError.__name__][ TuneRequestErrorCodes. REQ_ERR_RETRY_EXHAUSTED] = TuneRequestModuleError( error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED) exceptions_dict[TuneRequestModuleError.__name__][ TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_VALUE] = TuneRequestModuleError( error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE) return exceptions_dict
def is_retry_non_tune_ex(self, _tries, ex, request_label, request_url): is_retry = True raised_exception = None error_exception = ex ex_extra = { 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label } if not self.request_retry_excps_func or \ not self.request_retry_excps_func(error_exception, request_label): self.logger.error( "Request Retry: Unexpected: {}: Not Retry Candidate".format(base_class_name(error_exception)), extra=ex_extra ) is_retry = False raised_exception = error_exception if is_retry: self.logger.warning( "Request Retry: Unexpected: {}: Retry Candidate".format(base_class_name(error_exception)), extra=ex_extra ) if self.is_exhausted_retries(_tries, lambda: None): is_retry = False raised_exception = TuneRequestModuleError( error_message="Unexpected: {}".format(base_class_name(error_exception)), errors=error_exception, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED ) return is_retry, raised_exception
def is_retry_non_tune_ex(self, tries, ex, request_url, request_label=None): """Is Retry Non-TUNE Exception :param tries: :param ex: :param request_url: :param request_label: :return: """ _request_label = 'Is Retry Non-TUNE Exception' request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label is_retry = True raised_exception = None error_exception = ex ex_extra = { 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label } if not self.request_retry_excps_func or \ not self.request_retry_excps_func(error_exception, request_label): self.logger.error( '{}: Unexpected: {}: Not Retry Candidate'.format(request_label, base_class_name(error_exception)), extra=ex_extra ) is_retry = False raised_exception = error_exception if is_retry: self.logger.warning( '{}: Unexpected: {}: Retry Candidate'.format(request_label, base_class_name(error_exception)), extra=ex_extra ) if self.is_exhausted_retries(tries, lambda: None): is_retry = False raised_exception = TuneRequestModuleError( error_message='{}: Unexpected: {}'.format(request_label, base_class_name(error_exception)), errors=error_exception, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED ) return is_retry, raised_exception
def _request_data( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_headers=None, request_auth=None, cookie_payload=None, request_label=None, timeout=60, build_request_curl=True, allow_redirects=True, verify=True, stream=False ): """Request Data from requests. Args: request_method: request_method for the new :class:`Request` object. logger: logging instance request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. timeout: (optional) How long to wait for the server to send data before giving up. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. Returns: requests.Response """ if request_label is None: request_label = 'Request Data' if not request_method: raise TuneRequestValueError(error_message="Parameter 'request_method' not defined") if not request_url: raise TuneRequestValueError(error_message="Parameter 'request_url' not defined") self.built_request_curl = None self.logger.debug( '{}: Session: Details'.format(request_label), extra={'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label} ) response = None headers = None if request_headers: headers = request_headers request_method = request_method.upper() if request_data and isinstance(request_data, str): if len(request_data) <= 20: request_data_extra = request_data else: request_data_extra = request_data[:20] + ' ...' else: request_data_extra = safe_str(request_data) request_extra = { 'request_method': request_method, 'request_url': request_url, 'timeout': timeout, 'request_params': safe_dict(request_params), 'request_data': request_data_extra, 'request_headers': safe_dict(headers), 'request_label': request_label } self.logger.debug('{}: Details'.format(request_label), extra=request_extra) self.built_request_curl = None kwargs = {} if headers: kwargs.update({'headers': headers}) if request_auth: kwargs.update({'auth': request_auth}) if timeout and isinstance(timeout, int): kwargs.update({'timeout': timeout}) if allow_redirects: kwargs.update({'allow_redirects': allow_redirects}) if stream: kwargs.update({'stream': stream}) if cookie_payload: kwargs.update({'cookies': cookie_payload}) kwargs.update({'verify': verify}) try: if build_request_curl: # In case no authentication information has been provided, # use session's cookies information, if exists if not request_auth and self.session.cookies and len(self.session.cookies) > 0: request_auth = self.session.cookies self.built_request_curl = command_line_request_curl( request_method=request_method, request_url=request_url, request_headers=headers, request_params=request_params, request_data=request_data, request_json=request_json, request_auth=request_auth, request_timeout=timeout, request_allow_redirects=allow_redirects ) self.logger.debug( '{}: Curl'.format(request_label), extra={ 'request_method': request_method, 'request_label': request_label, 'request_curl': self.built_request_curl } ) if hasattr(response, 'url'): self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url}) if request_params: kwargs.update({'params': request_params}) if request_data: kwargs.update({'data': request_data}) if request_json: kwargs.update({'json': request_json}) if headers: kwargs.update({'headers': headers}) kwargs.update({'request_method': request_method, 'request_url': request_url}) response = self.tune_request.request(**kwargs) except Exception as ex: self.logger.error( '{}: Request Base: Error'.format(request_label), extra={ 'request_label': request_label, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) } ) raise if response is None: self.logger.error( '{}: Response: Failed'.format(request_label), extra={'request_curl': self.built_request_curl}, ) raise TuneRequestModuleError( error_message='{}: Response: Failed'.format(request_label), error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE, error_request_curl=self.built_request_curl ) http_status_code = response.status_code response_headers = json.loads(json.dumps(dict(response.headers))) http_status_type = \ http_status_code_to_type(http_status_code) http_status_desc = \ http_status_code_to_desc(http_status_code) response_extra = { 'http_status_code': http_status_code, 'http_status_type': http_status_type, 'http_status_desc': http_status_desc, 'response_headers': safe_dict(response_headers), } self.logger.debug( '{}: Response: Details'.format(request_label), extra=response_extra, ) http_status_successful = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL ) http_status_redirection = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION ) if http_status_successful or http_status_redirection: if hasattr(response, 'url') and \ response.url and \ len(response.url) > 0: response_extra.update({'response_url': response.url}) self.logger.debug( '{}: Cookie Payload'.format(request_label), extra={'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label} ) assert response return response else: response_extra.update({'error_request_curl': self.built_request_curl}) self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra) json_response_error = \ build_response_error_details( response=response, request_label=request_label, request_url=request_url ) extra_error = copy.deepcopy(json_response_error) if self.logger_level == logging.INFO: error_response_details = \ extra_error.get('response_details', None) if error_response_details and \ isinstance(error_response_details, str) and \ len(error_response_details) > 100: extra_error['response_details'] = error_response_details[:100] + ' ...' if self.built_request_curl and \ 'error_request_curl' not in extra_error: extra_error.update({'error_request_curl': self.built_request_curl}) self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error) kwargs = { 'error_status': json_response_error.get("response_status", None), 'error_reason': json_response_error.get("response_reason", None), 'error_details': json_response_error.get("response_details", None), 'error_request_curl': self.built_request_curl } if http_status_code in [ HttpStatusCode.BAD_REQUEST, HttpStatusCode.UNAUTHORIZED, HttpStatusCode.FORBIDDEN, HttpStatusCode.NOT_FOUND, HttpStatusCode.METHOD_NOT_ALLOWED, HttpStatusCode.NOT_ACCEPTABLE, HttpStatusCode.REQUEST_TIMEOUT, HttpStatusCode.CONFLICT, HttpStatusCode.GONE, HttpStatusCode.UNPROCESSABLE_ENTITY, HttpStatusCode.TOO_MANY_REQUESTS, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestClientError(**kwargs) if http_status_code in [ HttpStatusCode.INTERNAL_SERVER_ERROR, HttpStatusCode.NOT_IMPLEMENTED, HttpStatusCode.BAD_GATEWAY, HttpStatusCode.SERVICE_UNAVAILABLE, HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestServiceError(**kwargs) kwargs.update({'error_code': json_response_error['response_status_code']}) extra_unhandled = copy.deepcopy(kwargs) extra_unhandled.update({'http_status_code': http_status_code}) self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled) raise TuneRequestModuleError(**kwargs)
def try_send_request(self, attempts, tries, request_func, request_retry_func, request_url, request_label=None): """Try Send Request :param attempts: :param tries: :param request_func: :param request_retry_func: :param request_url: :param request_label: :return: """ _request_label = 'Try Send Request' request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label to_raise_exception = None to_return_response = None try: response = request_func() if response is None: raise TuneRequestModuleError( error_message='{}: No response'.format(request_label), error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE ) if self.is_return_response(request_retry_func, request_url, response, request_label=request_label): to_return_response = response else: self.logger.debug( '{}: Response: Valid: Retry Candidate'.format(request_label), extra={'request_url': request_url, 'request_label': request_label} ) except tuple(self.request_retry_excps) as retry_ex: if not self.is_retry_retry_ex(tries, request_url, retry_ex, request_label=request_label): to_raise_exception = retry_ex except TuneRequestBaseError as tmv_ex: if not self.is_retry_non_retry_ex(tries, tmv_ex, request_label=request_label): to_raise_exception = tmv_ex except Exception as ex: is_retry, raised_exception = self.is_retry_non_tune_ex(tries, ex, request_url, request_label=request_label) if not is_retry: to_raise_exception = raised_exception # A final check, whether we need to raise an exception, is in case the number of retries has exhausted. if not to_raise_exception and self.is_exhausted_retries( tries, partial( self.logger.error, '{}: Exhausted Retries'.format(request_label), extra={ 'attempts': attempts, 'tries': tries, 'request_url': request_url, 'request_label': request_label } ) ): to_raise_exception = TuneRequestModuleError( error_message=('{}: Exhausted Retries: {}').format(request_label, request_url), error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED ) return to_raise_exception, to_return_response
def request( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_retry=None, request_retry_excps=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_headers=None, request_auth=None, cookie_payload=None, build_request_curl=True, allow_redirects=True, verify=True, stream=False, request_label=None ): """Request data from remote source with retries. Args: request_method: request_method for the new :class:`Request` object. request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_retry: (optional) Retry configuration. request_retry_func: (optional) Retry function, alternative to request_retry_excps. request_retry_excps: An exception or a tuple of exceptions to catch. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. request_label: Returns: requests.Response: Data result if success or None if error. Raises: ServiceGatewayTimeoutError: Upon any timeout condition. Exception: Upon error within this request_method. Notes: * tries: the maximum number of attempts. default: 1. * delay: initial delay between attempts. default: 1. * max_delay: the maximum value of delay. default: None (no limit). * backoff: multiplier applied to delay between attempts. default: 1 (no backoff). * jitter: extra seconds added to delay between attempts. default: 0. """ if request_label is None: request_label = 'Request' self.logger.debug('{}: Start'.format(request_label)) timeout = None if not verify: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) if request_method: request_method = request_method.upper() if request_label is None: request_label = 'Request' if not request_retry: request_retry = {} if 'timeout' not in request_retry: request_retry['timeout'] = self._REQUEST_CONFIG['timeout'] if 'tries' not in request_retry: request_retry['tries'] = self._REQUEST_CONFIG['tries'] if 'delay' not in request_retry: request_retry['delay'] = self._REQUEST_CONFIG['delay'] self._prep_request_retry(request_retry, request_retry_http_status_codes) if not self.tune_request: self.tune_request = TuneRequest( retry_tries=self.retry_tries, retry_backoff=self.retry_backoff, retry_codes=self.request_retry_http_status_codes ) key_user_agent = 'User-Agent' header_user_agent = {key_user_agent: __USER_AGENT__} if request_headers: if key_user_agent not in request_headers: request_headers.update(header_user_agent) else: request_headers = header_user_agent kwargs = { 'request_method': request_method, 'request_url': request_url, 'request_params': request_params, 'request_data': request_data, 'request_json': request_json, 'request_headers': request_headers, 'request_auth': request_auth, 'cookie_payload': cookie_payload, 'request_label': request_label, 'timeout': timeout, 'build_request_curl': build_request_curl, 'allow_redirects': allow_redirects, 'verify': verify, 'stream': stream } time_start_req = dt.datetime.now() if request_retry_func is None: request_retry_func = self.request_retry_func if request_retry_excps_func is None: request_retry_excps_func = self.request_retry_excps_func if request_retry_excps_func is None: request_retry_excps_func = mv_request_retry_excps_func if request_retry_http_status_codes is not None: self.request_retry_http_status_codes = request_retry_http_status_codes if request_retry_excps is not None: self.request_retry_excps = request_retry_excps extra_request = copy.copy(kwargs) if request_retry: extra_request.update({'request_retry': request_retry}) if request_label: extra_request.update({'request_label': request_label}) if request_retry: extra_request.update({'request_retry': request_retry}) if request_retry_func: extra_request.update({'request_retry_func': request_retry_func}) if request_retry_http_status_codes: extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes}) if request_retry_excps: extra_request.update({'request_retry_excps': request_retry_excps}) if request_retry_excps: extra_request.update({'request_retry_excps_func': request_retry_excps_func}) extra_request.update(env_usage()) self.logger.debug('{}: Start: Details'.format(request_label), extra=extra_request) try: self._prep_request_retry(request_retry, request_retry_http_status_codes) response = self._request_retry( call_func=self._request_data, fargs=None, fkwargs=kwargs, timeout=timeout, request_label=request_label, request_retry_func=request_retry_func, request_retry_excps_func=request_retry_excps_func ) except ( requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout, ) as ex_req_timeout: raise TuneRequestServiceError( error_message='{}: Exception: Timeout'.format(request_label), errors=ex_req_timeout, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.HTTPError as ex_req_http: raise TuneRequestModuleError( error_message='{}: Exception: HTTP Error'.format(request_label), errors=ex_req_http, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_HTTP ) except ( requests.exceptions.ConnectionError, requests.exceptions.ProxyError, requests.exceptions.SSLError, ) as ex_req_connect: raise TuneRequestModuleError( error_message='{}: Exception: {}'.format(request_label, base_class_name(ex_req_connect)), errors=ex_req_connect, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except (BrokenPipeError, ConnectionError,) as ex_ose_connect: raise TuneRequestModuleError( error_message='{}: Exception: OSE {}'.format(request_label, base_class_name(ex_ose_connect)), errors=ex_ose_connect, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_CONNECT ) except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol: raise TuneRequestModuleError( error_message='{}: Exception: Urllib3: Protocol Error'.format(request_label), errors=ex_req_urllib3_protocol, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout: raise TuneRequestServiceError( error_message='{}: Exception: Urllib3: Read Timeout Error'.format(request_label), errors=ex_req_urllib3_read_timeout, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.TooManyRedirects as ex_req_redirects: raise TuneRequestModuleError( error_message='{}: Exception: Too Many Redirects'.format(request_label), errors=ex_req_redirects, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_REDIRECTS ) except requests.exceptions.RetryError as ex_req_adapter_retry: # Expected structure of RetryError: # RetryError has list 'args' of len 1: RetryError.args[0] == MaxRetryError (from urlib3) # Structure of MaxRetryError: MaxRetryError.reason == ResponseError (from urlib3) if len(ex_req_adapter_retry.args) == 1 and hasattr(ex_req_adapter_retry.args[0], 'reason'): response_err = ex_req_adapter_retry.args[0].reason status_codes = [int(s) for s in response_err.args[0].split() if s.isdigit()] if len(status_codes) == 1: http_status_code = status_codes[0] http_status_type = http_status_code_to_type(http_status_code) error_kwargs = { 'errors': ex_req_adapter_retry, 'error_code': http_status_code, 'error_reason': response_err.args[0], 'error_message': "Request: Exception: HTTPAdapter: Retries exhausted on: '{}'" .format(request_url), 'error_request_curl': self.built_request_curl, } self.logger.error( '{}: Exception: HTTPAdapter: Max retry error'.format(request_label), extra=error_kwargs ) if http_status_type == HttpStatusType.CLIENT_ERROR: raise TuneRequestClientError(**error_kwargs) elif http_status_type == HttpStatusType.SERVER_ERROR: raise TuneRequestServiceError(**error_kwargs) # THIS BLOCK SHOULD NOT BE ACTUALLY ACCESSED. IF IT DOES LOOK INTO IT: self.logger.error( '{}: Unexpected RetryError occurred'.format(request_label), extra={'request_curl': self.built_request_curl} ) raise TuneRequestModuleError( error_message='{}: Exception: HTTPAdapter: Unexpected Retry Error'.format(request_label), errors=ex_req_adapter_retry, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED, ) except requests.exceptions.RequestException as ex_req_request: raise TuneRequestModuleError( error_message='{}: Exception: Request Error'.format(request_label), errors=ex_req_request, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST ) except TuneRequestBaseError: raise except Exception as ex: print_traceback(ex) raise TuneRequestModuleError( error_message='{}: Exception: Unexpected'.format(request_label), errors=ex, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE ) time_end_req = dt.datetime.now() diff_req = time_end_req - time_start_req request_time_msecs = int(diff_req.total_seconds() * 1000) self.logger.info( '{}: Finished'.format(request_label), extra={ 'request_time_msecs': request_time_msecs, } ) self.logger.debug('{}: Usage'.format(request_label), extra=env_usage()) return response
def validate_response( response, request_curl, request_label=None, ): """Validate response Args: response: request_label: request_url: Returns: """ response_extra = {} if request_label: response_extra.update({'request_label': request_label}) if not response: log.error("Validate Response: Failed: None", extra=response_extra) raise TuneRequestModuleError( error_message="Validate Response: Failed: None", error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) else: log.debug("Validate Response: Defined", extra=response_extra) response_extra.update({'http_status_code': response.status_code}) if hasattr(response, 'text'): response_text_length = len(response.text) response_extra.update({'response_text_length': response_text_length}) if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_extra.update( {'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_extra.update({ 'Content-Length': convert_size(response_headers_content_length) }) if 'Content-Encoding' in response.headers: response_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_extra.update( {'Content-Encoding': response_content_encoding}) if 'Transfer-Encoding' in response.headers: response_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_extra.update( {'Transfer-Encoding': response_transfer_encoding}) if not is_http_status_successful(http_status_code=response.status_code): log.error("Validate Response: Failed", extra=response_extra) raise TuneRequestModuleError( error_message="Validate Request: Failed", error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) else: log.debug("Validate Response: Success", extra=response_extra)
def handle_json_decode_error( response_decode_ex, response, response_extra=None, request_label=None, request_curl=None, ): """Handle JSON Decode Error Args: response_json_decode_error: response: response_extra: request_label: Returns: """ if response_extra is None: response_extra = {} if request_label: response_extra.update({'request_label': request_label}) if hasattr(response, 'text') and \ response.text and \ len(response.text) > 0: response_details = response.text response_details_source = 'text' response_content_length = len(response_details) if response_details.startswith('<html'): response_details_source = 'html' soup_html = bs4.BeautifulSoup(response_details, "html.parser") # kill all script and style elements for script in soup_html(["script", "style"]): script.extract() # rip it out text_html = soup_html.get_text() lines_html = [ line for line in text_html.split('\n') if line.strip() != '' ] lines_html = [line.strip(' ') for line in lines_html] response_details = lines_html elif response_details.startswith('<?xml'): response_details_source = 'xml' response_details = json.dumps(xmltodict.parse(response_details)) else: pprint(response_details) response_extra.update({ 'response_details': response_details, 'response_details_source': response_details_source, 'response_content_length': response_content_length, 'error_exception': base_class_name(response_decode_ex), 'error_details': get_exception_message(response_decode_ex) }) log.error("Validate JSON Response: Failed: Invalid", extra=response_extra) raise TuneRequestModuleError( error_message="Validate JSON Response: Failed: Invalid", errors=response_decode_ex, error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
def requests_response_json( response, request_curl, request_label=None, raise_ex_if_not_json_response=True, ): """Get JSON from response from requests Args: response: request_label: Returns: """ json_response = None response_extra = {} if request_label: response_extra.update({'request_label': request_label}) try: json_response = response.json() response_details_source = 'json' response_content_length = len(json_response) response_extra.update({ 'response_details_source': response_details_source, 'response_content_length': response_content_length }) except json.decoder.JSONDecodeError as json_decode_ex: log.error("Validate JSON Response: Failed: JSONDecodeError", extra=response_extra) data = requests_toolbelt.utils.dump.dump_all(response) pprint(data.decode('utf-8')) pprint(response.text) handle_json_decode_error(response_decode_ex=json_decode_ex, response=response, response_extra=response_extra, request_label=request_label, request_curl=request_curl) except Exception as ex: log.error("Validate JSON Response: Failed: Exception", extra=response_extra) pprint(response.text) handle_json_decode_error(response_decode_ex=ex, response=response, response_extra=response_extra, request_label=request_label, request_curl=request_curl) if json_response is None: if raise_ex_if_not_json_response: log.error("Validate JSON Response: Failed: None", extra=response_extra) raise TuneRequestModuleError( error_message="Validate JSON Response: Failed: None", error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) else: log.warning("Validate JSON Response: None", extra=response_extra) else: log.debug("Validate JSON Response: Valid", extra=response_extra) return json_response
def validate_json_response(response, request_curl, request_label=None, response_content_type_expected='application/json', raise_ex_if_not_json_response=True): """Validate JSON response. Args: response: request_label: response_content_type_expected: raise_ex_if_not_json_response: Returns: """ validate_response(response, request_label) json_response = None response_extra = {} if request_label: response_extra.update({'request_label': request_label}) response_extra.update( {'Content-Type (Expected)': response_content_type_expected}) if hasattr(response, 'headers'): response_content_type = response.headers.get('Content-Type', None) if response_content_type is not None: is_valid_response_content_type = \ response_content_type == response_content_type_expected or \ response_content_type.startswith(response_content_type_expected) if is_valid_response_content_type: json_response = requests_response_json( response=response, request_curl=request_curl, request_label=request_label, ) elif response_content_type.startswith('text/html'): try: response_content_html_lines = \ requests_response_text_html( response=response ) except Exception as ex: raise TuneRequestModuleError( error_message=request_label, errors=ex, error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) raise TuneRequestModuleError( error_message="Unexpected 'Content-Type': '{}', Expected: '{}'" .format(response_content_type, response_content_type_expected), errors=response_content_html_lines, error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) else: raise TuneRequestModuleError( error_message="Unexpected 'Content-Type': '{}', Expected: '{}'" .format(response_content_type, response_content_type_expected), error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) else: raise TuneRequestModuleError(error_message="Undefined 'Content-Type'", error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) response_extra.update({ 'http_status_code': response.status_code, 'raise_ex_if_not_json_response': raise_ex_if_not_json_response }) log.debug("Validate JSON Response: Details", extra=response_extra) return json_response
def request( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_retry=None, request_retry_excps=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_headers=None, request_auth=None, cookie_payload=None, build_request_curl=True, allow_redirects=True, verify=True, stream=False, request_label=None ): """Request data from remote source with retries. Args: request_method: request_method for the new :class:`Request` object. request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_retry: (optional) Retry configuration. request_retry_func: (optional) Retry function, alternative to request_retry_excps. request_retry_excps: An exception or a tuple of exceptions to catch. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. request_label: Returns: requests.Response: Data result if success or None if error. Raises: ServiceGatewayTimeoutError: Upon any timeout condition. Exception: Upon error within this request_method. Notes: * tries: the maximum number of attempts. default: 1. * delay: initial delay between attempts. default: 1. * max_delay: the maximum value of delay. default: None (no limit). * backoff: multiplier applied to delay between attempts. default: 1 (no backoff). * jitter: extra seconds added to delay between attempts. default: 0. """ self.logger.debug("Request: Start: {}".format(request_label if request_label else "")) timeout = None retry_tries = None retry_delay = None retry_backoff = 0 retry_jitter = 0 retry_max_delay = None if not verify: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) if request_method: request_method = request_method.upper() if not request_retry: request_retry = {} if 'timeout' not in request_retry: request_retry['timeout'] = self._REQUEST_CONFIG['timeout'] if 'tries' not in request_retry: request_retry['tries'] = self._REQUEST_CONFIG['tries'] if 'delay' not in request_retry: request_retry['delay'] = self._REQUEST_CONFIG['delay'] self._prep_request_retry(request_retry, request_retry_http_status_codes) if not self.tune_request: self.tune_request = TuneRequest( retry_tries=self.retry_tries, retry_backoff=self.retry_backoff, retry_codes=self.request_retry_http_status_codes ) key_user_agent = 'User-Agent' header_user_agent = {key_user_agent: __USER_AGENT__} if request_headers: if key_user_agent not in request_headers: request_headers.update(header_user_agent) else: request_headers = header_user_agent kwargs = { 'request_method': request_method, 'request_url': request_url, 'request_params': request_params, 'request_data': request_data, 'request_json': request_json, 'request_headers': request_headers, 'request_auth': request_auth, 'cookie_payload': cookie_payload, 'request_label': request_label, 'timeout': timeout, 'build_request_curl': build_request_curl, 'allow_redirects': allow_redirects, 'verify': verify, 'stream': stream } time_start_req = dt.datetime.now() if request_retry_func is None: request_retry_func = self.request_retry_func if request_retry_excps_func is None: request_retry_excps_func = self.request_retry_excps_func if request_retry_http_status_codes is not None: self.request_retry_http_status_codes = request_retry_http_status_codes if request_retry_excps is not None: self.request_retry_excps = request_retry_excps extra_request = copy.copy(kwargs) if request_retry: extra_request.update({'request_retry': request_retry}) if request_label: extra_request.update({'request_label': request_label}) if request_retry: extra_request.update({'request_retry': request_retry}) if request_retry_func: extra_request.update({'request_retry_func': request_retry_func}) if request_retry_http_status_codes: extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes}) if request_retry_excps: extra_request.update({'request_retry_excps': request_retry_excps}) if request_retry_excps: extra_request.update({'request_retry_excps_func': request_retry_excps_func}) self.logger.debug("Request: Details", extra=extra_request) try: self._prep_request_retry(request_retry, request_retry_http_status_codes) response = self._request_retry( call_func=self._request, fargs=None, fkwargs=kwargs, timeout=timeout, request_label=request_label, request_retry_func=request_retry_func, request_retry_excps_func=request_retry_excps_func ) except ( requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout, ) as ex_req_timeout: raise TuneRequestServiceError( error_message="Request: Exception: Timeout", errors=ex_req_timeout, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.HTTPError as ex_req_http: raise TuneRequestModuleError( error_message="Request: Exception: HTTP Error", errors=ex_req_http, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_HTTP ) except requests.exceptions.ConnectionError as ex_req_connect: raise TuneRequestModuleError( error_message="Request: Exception: Connection Error", errors=ex_req_connect, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except BrokenPipeError as ex_broken_pipe: raise TuneRequestModuleError( error_message="Request: Exception: Broken Pipe Error", errors=ex_broken_pipe, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except ConnectionError as ex_connect: raise TuneRequestModuleError( error_message="Request: Exception: Connection Error", errors=ex_connect, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol: raise TuneRequestModuleError( error_message="Request: Exception: Protocol Error", errors=ex_req_urllib3_protocol, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT ) except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout: raise TuneRequestServiceError( error_message="Request: Exception: Urllib3: Read Timeout Error", errors=ex_req_urllib3_read_timeout, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT ) except requests.exceptions.TooManyRedirects as ex_req_redirects: raise TuneRequestModuleError( error_message="Request: Exception: Too Many Redirects", errors=ex_req_redirects, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_REDIRECTS ) except requests.exceptions.RequestException as ex_req_request: raise TuneRequestModuleError( error_message="Request: Exception: Request Error", errors=ex_req_request, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST ) except TuneRequestBaseError: raise except Exception as ex: print_traceback(ex) raise TuneRequestModuleError( error_message="Request: Exception: Unexpected", errors=ex, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE ) time_end_req = dt.datetime.now() diff_req = time_end_req - time_start_req request_time_msecs = int(diff_req.total_seconds() * 1000) self.logger.debug( "Request: Completed", extra={'request_label': request_label, 'request_time_msecs': request_time_msecs} ) return response
def _request_retry( self, call_func, fargs=None, fkwargs=None, timeout=60, request_retry_func=None, request_retry_excps_func=None, request_label=None, ): """Request Retry Args: call_func: the function to execute. request_retry_excps: A tuple of exceptions to catch. fargs: the positional arguments of the function to execute. fkwargs: the named arguments of the function to execute. timeout: (optional) How long to wait for the server to send data before giving up. request_retry_func: (optional) Retry alternative to request_retry_excps. retry_tries: the maximum number of attempts. default: -1 (infinite). retry_delay: initial delay between attempts. default: 0. retry_max_delay:the maximum value of delay. default: None (no limit). retry_backoff: multiplier applied to delay between attempts. default: 1 (no backoff). retry_jitter: extra seconds added to delay between attempts. default: 0. request_label: Label Returns: """ request_retry_extra = {'timeout': timeout} request_retry_extra.update({ 'request_retry_http_status_codes': self.request_retry_http_status_codes }) if self.request_retry_excps is not None: request_retry_excp_names = [ excp.__name__ for excp in list(self.request_retry_excps) ] request_retry_extra.update( {'request_retry_excps': request_retry_excp_names}) if request_retry_func is not None: request_retry_func_name = request_retry_func.__name__ request_retry_extra.update( {'request_retry_func': request_retry_func_name}) if request_retry_excps_func is not None: request_retry_excps_func_name = request_retry_excps_func.__name__ request_retry_extra.update( {'request_retry_excps_func': request_retry_excps_func_name}) self.logger.debug("Request Retry: Start: {}".format( request_label if request_label else ""), extra=request_retry_extra) args = fargs if fargs else list() kwargs = fkwargs if fkwargs else dict() request_url = kwargs[ 'request_url'] if kwargs and 'request_url' in kwargs else "" _attempts = 0 _tries, _delay, _timeout = self.retry_tries, self.retry_delay, self.timeout while _tries: _attempts += 1 fkwargs['timeout'] = _timeout request_func = partial(call_func, *args, **kwargs) self.logger.debug("Request Retry: Attempt: {}: {}".format( request_label if request_label else "", _attempts), extra={ 'attempts': _attempts, 'timeout': _timeout, 'tries': _tries, 'delay': _delay, 'request_url': request_url }) error_exception = None _tries -= 1 try: response = request_func() if response is None: raise TuneRequestModuleError( error_message="Request Retry: No response", error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_VALUE) self.logger.debug("Request Retry: Checking Response", extra={ 'request_url': request_url, 'request_label': request_label }) if request_retry_func is not None: if not request_retry_func(response): self.logger.debug( "Request Retry: Response: Valid: Not Retry Candidate", extra={ 'request_url': request_url, 'request_label': request_label }) return response else: self.logger.debug("Request Retry: Response: Valid", extra={ 'request_url': request_url, 'request_label': request_label }) return response self.logger.debug( "Request Retry: Response: Valid: Retry Candidate", extra={ 'request_url': request_url, 'request_label': request_label }) except self.request_retry_excps as retry_ex: error_exception = retry_ex self.logger.warning( "Request Retry: Expected: {}: Retry Candidate".format( base_class_name(error_exception)), extra={ 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label }) if not _tries: self.logger.error( "Request Retry: Expected: {}: Exhausted Retries". format(base_class_name(error_exception))) raise except TuneRequestBaseError as tmv_ex: error_exception = tmv_ex tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({ 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_label': request_label }) if not self.request_retry_excps_func or \ not self.request_retry_excps_func(tmv_ex, request_label): self.logger.error( "Request Retry: Integration: {}: Not Retry Candidate". format(base_class_name(error_exception)), extra=tmv_ex_extra) raise self.logger.warning( "Request Retry: Integration: {}: Retry Candidate".format( base_class_name(error_exception)), extra=tmv_ex_extra) if not _tries: self.logger.error( "Request Retry: Integration: {}: Exhausted Retries". format(base_class_name(error_exception))) raise except Exception as ex: error_exception = ex ex_extra = { 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label } if not self.request_retry_excps_func or \ not self.request_retry_excps_func(error_exception, request_label): self.logger.error( "Request Retry: Unexpected: {}: Not Retry Candidate". format(base_class_name(error_exception)), extra=ex_extra) raise self.logger.warning( "Request Retry: Unexpected: {}: Retry Candidate".format( base_class_name(error_exception)), extra=ex_extra) if not _tries: raise TuneRequestModuleError( error_message="Unexpected: {}".format( base_class_name(error_exception)), errors=error_exception, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_RETRY_EXHAUSTED) if not _tries: self.logger.error("Request Retry: Exhausted Retries", extra={ 'attempts': _attempts, 'tries': _tries, 'request_url': request_url, 'request_label': request_label }) raise TuneRequestModuleError( error_message=( "Request Retry: Exhausted Retries: {}: {}").format( request_label, request_url), error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED) self.logger.info("Request Retry: Performing Retry", extra={ 'tries': _tries, 'delay': _delay, 'timeout': _timeout, 'request_url': request_url, 'request_label': request_label }) time.sleep(_delay) if self.retry_backoff and self.retry_backoff > 0: _delay *= self.retry_backoff if self.retry_jitter and self.retry_jitter > 0: _delay += self.retry_jitter if self.retry_max_delay is not None: _delay = min(_delay, self.retry_max_delay)
def validate_response( response, request_curl, request_label=None, ): """Validate response Args: response: request_curl: request_label: Returns: """ response_extra = {} if request_label is None: request_label = 'Validate Response' if not response: error_message = f'{request_label}: Failed: None' log.error(error_message, extra=response_extra) raise TuneRequestModuleError( error_message=error_message, error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) log.debug(f'{request_label}: Defined', extra=response_extra) response_extra.update({'http_status_code': response.status_code}) # Not using hasattr on purpose # Assuming positive approach will give us speedup since when attribute exists # hasattr takes double the time. # Anyway we use text attribute here to logging purpose only try: response_extra.update({'response_text_length': len(response.text)}) except AttributeError: pass if response.headers: if 'Content-Type' in response.headers: response_headers_content_type = \ safe_str(response.headers['Content-Type']) response_extra.update( {'Content-Type': response_headers_content_type}) if 'Content-Length' in response.headers: response_headers_content_length = \ safe_int(response.headers['Content-Length']) response_extra.update({ 'Content-Length': bytes_to_human(response_headers_content_length) }) if 'Content-Encoding' in response.headers: response_content_encoding = \ safe_str(response.headers['Content-Encoding']) response_extra.update( {'Content-Encoding': response_content_encoding}) if 'Transfer-Encoding' in response.headers: response_transfer_encoding = \ safe_str(response.headers['Transfer-Encoding']) response_extra.update( {'Transfer-Encoding': response_transfer_encoding}) if not is_http_status_successful(http_status_code=response.status_code): error_message = f'{request_label}: Failed' log.error(error_message, extra=response_extra) raise TuneRequestModuleError( error_message=error_message, error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE) log.debug(f'{request_label}: Success', extra=response_extra)
def validate_json_response(response, request_curl, request_label=None, response_content_type_expected='application/json', raise_ex_if_not_json_response=True): """Validate JSON response. Args: response: request_curl request_label: response_content_type_expected: raise_ex_if_not_json_response: Returns: """ validate_response(response, request_curl, request_label) json_response = None response_extra = {} if request_label is None: request_label = 'Validate JSON Response' response_extra.update({'Content-Type': response_content_type_expected}) try: response_content_type = response.headers.get('Content-Type', None) if response.headers.get('Content-Type', None) is not None: is_valid_response_content_type = \ response_content_type == response_content_type_expected or \ response_content_type.startswith(response_content_type_expected) if is_valid_response_content_type: json_response = requests_response_json( response=response, request_curl=request_curl, request_label=request_label, ) elif response_content_type.startswith('text/html'): try: response_content_html_lines = \ requests_response_text_html( response=response ) except Exception as ex: raise TuneRequestModuleError( error_message=request_label, errors=ex, error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) raise TuneRequestModuleError( error_message= f"Content-Type: Expected: '{response_content_type_expected}', Actual: '{response_content_type}'", errors=response_content_html_lines, error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) else: raise TuneRequestModuleError( error_message= f"Content-Type: Expected: '{response_content_type_expected}', Actual: '{response_content_type}'", error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) except AttributeError: raise TuneRequestModuleError(error_message="Content-Type: Undefined", error_request_curl=request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED) response_extra.update({ 'http_status_code': response.status_code, 'raise_ex_if_not_json_response': raise_ex_if_not_json_response }) log.debug( f'{request_label}: Success: JSON', extra=response_extra, ) return json_response