def is_retry_non_retry_ex(self, _tries, request_label, tmv_ex): is_retry = True error_exception = tmv_ex tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({ 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_label': request_label }) self.logger.warning( "Request Retry: Failed: {}".format(get_exception_message(error_exception)), extra=tmv_ex.to_dict() ) if not self.request_retry_excps_func or \ not self.request_retry_excps_func(tmv_ex, request_label): tmv_ex_extra.update({'request_retry_excps_func': self.request_retry_excps_func}) self.logger.error( "Request Retry: Integration: {}: Not Retry Candidate".format(base_class_name(error_exception)), extra=tmv_ex_extra ) is_retry = False if is_retry: self.logger.warning( "Request Retry: Integration: {}: Retry Candidate".format(base_class_name(error_exception)), extra=tmv_ex_extra ) is_retry = not self.is_exhausted_retries( _tries, partial( self.logger.error, "Request Retry: Expected: {}: Exhausted Retries".format(base_class_name(error_exception)) ) ) return is_retry
def is_retry_retry_ex(self, tries, request_url, retry_ex, request_label=None): """Is Retry Retry Exception :param tries: :param request_url: :param retry_ex: :param request_label: :return: """ _request_label = 'Is Retry Retry Exception' request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label self.logger.warning( '{}: Expected: {}: Retry Candidate'.format(request_label, base_class_name(retry_ex)), extra={ 'error_details': get_exception_message(retry_ex), 'request_url': request_url, 'request_label': request_label } ) return not self.is_exhausted_retries( tries, partial( self.logger.error, '{}: Expected: {}: Exhausted Retries'.format(request_label, base_class_name(retry_ex)) ) )
def is_retry_non_tune_ex(self, _tries, ex, request_label, request_url): is_retry = True raised_exception = None error_exception = ex ex_extra = { 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label } if not self.request_retry_excps_func or \ not self.request_retry_excps_func(error_exception, request_label): self.logger.error( "Request Retry: Unexpected: {}: Not Retry Candidate".format(base_class_name(error_exception)), extra=ex_extra ) is_retry = False raised_exception = error_exception if is_retry: self.logger.warning( "Request Retry: Unexpected: {}: Retry Candidate".format(base_class_name(error_exception)), extra=ex_extra ) if self.is_exhausted_retries(_tries, lambda: None): is_retry = False raised_exception = TuneRequestModuleError( error_message="Unexpected: {}".format(base_class_name(error_exception)), errors=error_exception, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED ) return is_retry, raised_exception
def _upload_request_retry_excps_func(self, excp, request_label): """Upload Request Retry Exception Function Args: excp: Returns: """ error_exception = base_class_name(excp) error_details = get_exception_message(excp) if isinstance(excp, TuneRequestBaseError): log.debug("Request Retry: Upload Exception Func", extra={ 'request_label': request_label, 'error_exception': error_exception, 'error_details': error_details }) else: log.debug("Request Retry: Upload Exception Func: Unexpected", extra={ 'request_label': request_label, 'error_exception': error_exception, 'error_details': error_details }) if isinstance(excp, TuneRequestBaseError) and \ excp.error_code == TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT: if error_details.find('RemoteDisconnected') >= 0 or \ error_details.find('ConnectionResetError') >= 0: log.debug("Request Retry: Upload Exception Func: Retry", extra={ 'request_label': request_label, 'error_exception': error_exception, 'error_details': error_details }) return True if isinstance(excp, requests.exceptions.ConnectionError): if error_details.find('RemoteDisconnected') >= 0 or \ error_details.find('ConnectionResetError') >= 0: log.debug("Request Retry: Upload Exception Func: Retry", extra={ 'request_label': request_label, 'error_exception': error_exception, 'error_details': error_details }) return True log.debug("Request Retry: Upload Exception Func: Not Retry", extra={ 'request_label': request_label, 'error_exception': error_exception, 'error_details': error_details }) return False
def is_retry_non_retry_ex(self, tries, tmv_ex, request_label=None): """Is Retry Non-Retry Exception :param tries: :param tmv_ex: :param request_label: :return: """ _request_label = 'Is Retry Non-Retry Exception' request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label is_retry = True error_exception = tmv_ex tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({ 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_label': request_label }) self.logger.warning( '{}: Failed: {}'.format(request_label, get_exception_message(error_exception)), extra=tmv_ex.to_dict(), ) if not self.request_retry_excps_func or \ not self.request_retry_excps_func(tmv_ex, request_label): tmv_ex_extra.update({'request_retry_excps_func': self.request_retry_excps_func}) self.logger.error( '{}: Integration: {}: Not Retry Candidate'.format(request_label, base_class_name(error_exception)), extra=tmv_ex_extra ) is_retry = False if is_retry: self.logger.warning( '{}: Integration: {}: Retry Candidate'.format(request_label, base_class_name(error_exception)), extra=tmv_ex_extra ) is_retry = not self.is_exhausted_retries( tries, partial( self.logger.error, '{}: Expected: {}: Exhausted Retries'.format(request_label, base_class_name(error_exception)) ) ) return is_retry
def request(self, request_method, request_url, **kwargs): extra_session_request = {'method': request_method, 'url': request_url} extra_session_request.update(kwargs) log.debug("Session Request: Details", extra=extra_session_request) try: return self.session.request(method=request_method, url=request_url, **kwargs) except Exception as ex: log.warning( "Session Request: Failed: {}".format(get_exception_message(ex)), extra=extra_session_request, ) raise
def is_retry_retry_ex(self, tries, request_label, request_url, retry_ex): self.logger.warning( "Request Retry: Expected: {}: Retry Candidate".format(base_class_name(retry_ex)), extra={ 'error_details': get_exception_message(retry_ex), 'request_url': request_url, 'request_label': request_label } ) return not self.is_exhausted_retries( tries, partial( self.logger.error, "Request Retry: Expected: {}: Exhausted Retries".format(base_class_name(retry_ex)) ) )
def is_retry_non_tune_ex(self, tries, ex, request_url, request_label=None): """Is Retry Non-TUNE Exception :param tries: :param ex: :param request_url: :param request_label: :return: """ _request_label = 'Is Retry Non-TUNE Exception' request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label is_retry = True raised_exception = None error_exception = ex ex_extra = { 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label } if not self.request_retry_excps_func or \ not self.request_retry_excps_func(error_exception, request_label): self.logger.error( '{}: Unexpected: {}: Not Retry Candidate'.format(request_label, base_class_name(error_exception)), extra=ex_extra ) is_retry = False raised_exception = error_exception if is_retry: self.logger.warning( '{}: Unexpected: {}: Retry Candidate'.format(request_label, base_class_name(error_exception)), extra=ex_extra ) if self.is_exhausted_retries(tries, lambda: None): is_retry = False raised_exception = TuneRequestModuleError( error_message='{}: Unexpected: {}'.format(request_label, base_class_name(error_exception)), errors=error_exception, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED ) return is_retry, raised_exception
def request_safe(self, request_method, request_url, response_hook=None, exception_handler=None, **kwargs): response_hook, exception_handler = self.create_hooks( response_hook, exception_handler) try: return self.session.request(method=request_method, url=request_url, hooks={'response': response_hook}, **kwargs) except Exception as ex: log.warning("Session Request: Failed: {}".format( get_exception_message(ex)), extra={ 'request_method': request_method, 'request_url': request_url }) exception_handler(ex) return None
def request_upload_json_file(self, upload_request_url, upload_data_file_path, upload_data_file_size, is_upload_gzip, request_label, upload_timeout=None): """Upload File to requested URL. Args: upload_request_url: upload_data_file_path: upload_data_file_size: upload_timeout: Returns: """ request_retry_excps = REQUEST_RETRY_EXCPS request_retry_http_status_codes = REQUEST_RETRY_HTTP_STATUS_CODES upload_request_retry = {"timeout": 60, "tries": -1, "delay": 60} upload_request_headers = { 'Content-Length': '{}'.format(upload_data_file_size) } if is_upload_gzip: upload_request_headers.update({'Content-Type': 'application/gzip'}) else: upload_request_headers.update( {'Content-Type': 'application/json; charset=utf8'}) if upload_timeout: upload_request_retry["timeout"] = int(upload_timeout) upload_extra = { 'upload_request_url': upload_request_url, 'upload_data_file_path': upload_data_file_path, 'upload_data_file_size': upload_data_file_size, 'upload_request_retry': upload_request_retry, 'upload_request_headers': upload_request_headers } log.info("Request Upload JSON File: Start", extra=upload_extra) try: with open(upload_data_file_path, 'rb') as upload_fp: response = self.mv_request.request( request_method="PUT", request_url=upload_request_url, request_params=None, request_data=upload_fp, request_retry=upload_request_retry, request_headers=upload_request_headers, request_retry_excps=request_retry_excps, request_retry_http_status_codes= request_retry_http_status_codes, request_retry_excps_func=self. _upload_request_retry_excps_func, allow_redirects=False, build_request_curl=False, request_label=request_label) except TuneRequestBaseError as tmv_ex: tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({'error_exception': base_class_name(tmv_ex)}) log.error("Request Upload JSON File: Failed", extra=tmv_ex_extra) raise except Exception as ex: log.error("Request Upload JSON File: Failed: Unexpected", extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) }) print_traceback(ex) raise TuneRequestModuleError( error_message=( "Request Upload JSON File: Failed: Unexpected: {}: {}" ).format(base_class_name(ex), get_exception_message(ex)), errors=ex, error_code=TuneRequestErrorCodes.REQ_ERR_UPLOAD_DATA) log.info("Request Upload JSON File: Finished") return response
def download_csv(self, response, tmp_directory, tmp_csv_file_name, request_label=None, encoding_write=None, decode_unicode=False): self.logger.debug("Download CSV: Start") if not os.path.exists(tmp_directory): os.mkdir(tmp_directory) tmp_csv_file_path = \ "{tmp_directory}/{tmp_csv_file_name}".format( tmp_directory=tmp_directory, tmp_csv_file_name=tmp_csv_file_name ) if os.path.exists(tmp_csv_file_path): self.logger.debug("Removing previous CSV", extra={'file_path': tmp_csv_file_path}) os.remove(tmp_csv_file_path) mode_write = 'wb' if encoding_write is None else 'w' self.logger.debug("Download CSV: Details", extra={ 'file_path': tmp_csv_file_path, 'mode_write': mode_write, 'encoding_write': encoding_write, 'request_label': request_label }) chunk_total_sum = 0 with open(file=tmp_csv_file_path, mode=mode_write, encoding=encoding_write) as csv_file_wb: self.logger.debug("Download CSV: By Chunk: Started", extra={ 'file_path': tmp_csv_file_path, 'request_label': request_label }) error_exception = None error_details = None try: for chunk in response.iter_content( chunk_size=8192, decode_unicode=decode_unicode): if not chunk: break chunk_total_sum += 8192 csv_file_wb.write(chunk) csv_file_wb.flush() os.fsync(csv_file_wb.fileno()) self.logger.debug("Download CSV: By Chunk: Completed", extra={ 'file_path': tmp_csv_file_path, 'request_label': request_label }) except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex: error_exception = base_class_name(chunked_encoding_ex) error_details = get_exception_message(chunked_encoding_ex) self.logger.warning("Download CSV: ChunkedEncodingError", extra={ 'error_exception': error_exception, 'error_details': error_details, 'chunk_total_sum': convert_size(chunk_total_sum), 'request_label': request_label }) return (None, 0) except http_client.IncompleteRead as incomplete_read_ex: error_exception = base_class_name(incomplete_read_ex) error_details = get_exception_message(incomplete_read_ex) self.logger.warning("Download CSV: IncompleteRead", extra={ 'error_exception': error_exception, 'error_details': error_details, 'chunk_total_sum': convert_size(chunk_total_sum), 'request_label': request_label }) return (None, 0) except requests.exceptions.RequestException as request_ex: self.logger.error("Download CSV: Request Exception", extra={ 'error_exception': base_class_name(request_ex), 'error_details': get_exception_message(request_ex), 'chunk_total_sum': convert_size(chunk_total_sum), 'request_label': request_label }) raise except Exception as ex: self.logger.error("Download CSV: Unexpected Exception", extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex), 'chunk_total_sum': convert_size(chunk_total_sum), 'request_label': request_label }) raise tmp_csv_file_size = os.path.getsize(tmp_csv_file_path) bom_enc, bom_len, bom_header = detect_bom(tmp_csv_file_path) self.logger.debug("Download CSV: By Chunk: Completed: Details", extra={ 'file_path': tmp_csv_file_path, 'file_size': convert_size(tmp_csv_file_size), 'chunk_total_sum': convert_size(chunk_total_sum), 'bom_encoding': bom_enc }) tmp_csv_file_name_wo_ext = \ os.path.splitext( os.path.basename(tmp_csv_file_name) )[0] tmp_csv_file_path_wo_bom = \ "{tmp_directory}/{tmp_csv_file_name}_wo_bom.csv".format( tmp_directory=tmp_directory, tmp_csv_file_name=tmp_csv_file_name_wo_ext ) if os.path.exists(tmp_csv_file_path_wo_bom): os.remove(tmp_csv_file_path_wo_bom) bom_enc, bom_len = remove_bom(tmp_csv_file_path, tmp_csv_file_path_wo_bom) self.logger.debug("Download CSV: Encoding", extra={ 'bom_enc': bom_enc, 'bom_len': bom_len }) if bom_len > 0: tmp_csv_file_path = tmp_csv_file_path_wo_bom return (tmp_csv_file_path, tmp_csv_file_size)
def request_upload_data(self, upload_request_url, upload_data, upload_data_size, upload_timeout=None): """Upload Data to requested URL. Args: upload_request_url: upload_data: Returns: requests.Response """ log.info("Request Upload JSON Data: Start", extra={ 'upload_data_size': upload_data_size, 'upload_request_url': upload_request_url }) request_retry_excps = REQUEST_RETRY_EXCPS request_retry_http_status_codes = REQUEST_RETRY_HTTP_STATUS_CODES upload_request_retry = {"timeout": 60, "tries": -1, "delay": 60} request_headers = { 'Content-type': 'application/json; charset=utf8', 'Accept': 'text/plain', 'Content-Length': "{}".format(upload_data_size) } if upload_timeout: upload_request_retry["timeout"] = int(upload_timeout) try: response = self.mv_request.request( request_method="PUT", request_url=upload_request_url, request_params=None, request_data=upload_data, request_retry=upload_request_retry, request_retry_excps=request_retry_excps, request_retry_http_status_codes=request_retry_http_status_codes, request_retry_excps_func=self._upload_request_retry_excps_func, request_headers=request_headers, allow_redirects=False, build_request_curl=False, request_label="Upload Data to URL") except TuneRequestBaseError as tmv_ex: tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({'error_exception': base_class_name(tmv_ex)}) log.error("Upload: Failed", extra=tmv_ex_extra) raise except Exception as ex: print_traceback(ex) log.error("Upload: Failed: Unexpected", extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) }) raise TuneRequestModuleError( error_message=("RequestMvIntegration: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneRequestErrorCodes.REQ_ERR_UPLOAD_DATA) log.info("Request Upload JSON Data: Finished") return response
def _request_data( self, request_method, request_url, request_params=None, request_data=None, request_json=None, request_headers=None, request_auth=None, cookie_payload=None, request_label=None, timeout=60, build_request_curl=True, allow_redirects=True, verify=True, stream=False ): """Request Data from requests. Args: request_method: request_method for the new :class:`Request` object. logger: logging instance request_url: URL for the new :class:`Request` object. request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_json: (optional) json data to send in the body of the :class:`Request`. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. timeout: (optional) How long to wait for the server to send data before giving up. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. stream: (optional) if ``False``, the response content will be immediately downloaded. Returns: requests.Response """ if request_label is None: request_label = 'Request Data' if not request_method: raise TuneRequestValueError(error_message="Parameter 'request_method' not defined") if not request_url: raise TuneRequestValueError(error_message="Parameter 'request_url' not defined") self.built_request_curl = None self.logger.debug( '{}: Session: Details'.format(request_label), extra={'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label} ) response = None headers = None if request_headers: headers = request_headers request_method = request_method.upper() if request_data and isinstance(request_data, str): if len(request_data) <= 20: request_data_extra = request_data else: request_data_extra = request_data[:20] + ' ...' else: request_data_extra = safe_str(request_data) request_extra = { 'request_method': request_method, 'request_url': request_url, 'timeout': timeout, 'request_params': safe_dict(request_params), 'request_data': request_data_extra, 'request_headers': safe_dict(headers), 'request_label': request_label } self.logger.debug('{}: Details'.format(request_label), extra=request_extra) self.built_request_curl = None kwargs = {} if headers: kwargs.update({'headers': headers}) if request_auth: kwargs.update({'auth': request_auth}) if timeout and isinstance(timeout, int): kwargs.update({'timeout': timeout}) if allow_redirects: kwargs.update({'allow_redirects': allow_redirects}) if stream: kwargs.update({'stream': stream}) if cookie_payload: kwargs.update({'cookies': cookie_payload}) kwargs.update({'verify': verify}) try: if build_request_curl: # In case no authentication information has been provided, # use session's cookies information, if exists if not request_auth and self.session.cookies and len(self.session.cookies) > 0: request_auth = self.session.cookies self.built_request_curl = command_line_request_curl( request_method=request_method, request_url=request_url, request_headers=headers, request_params=request_params, request_data=request_data, request_json=request_json, request_auth=request_auth, request_timeout=timeout, request_allow_redirects=allow_redirects ) self.logger.debug( '{}: Curl'.format(request_label), extra={ 'request_method': request_method, 'request_label': request_label, 'request_curl': self.built_request_curl } ) if hasattr(response, 'url'): self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url}) if request_params: kwargs.update({'params': request_params}) if request_data: kwargs.update({'data': request_data}) if request_json: kwargs.update({'json': request_json}) if headers: kwargs.update({'headers': headers}) kwargs.update({'request_method': request_method, 'request_url': request_url}) response = self.tune_request.request(**kwargs) except Exception as ex: self.logger.error( '{}: Request Base: Error'.format(request_label), extra={ 'request_label': request_label, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) } ) raise if response is None: self.logger.error( '{}: Response: Failed'.format(request_label), extra={'request_curl': self.built_request_curl}, ) raise TuneRequestModuleError( error_message='{}: Response: Failed'.format(request_label), error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE, error_request_curl=self.built_request_curl ) http_status_code = response.status_code response_headers = json.loads(json.dumps(dict(response.headers))) http_status_type = \ http_status_code_to_type(http_status_code) http_status_desc = \ http_status_code_to_desc(http_status_code) response_extra = { 'http_status_code': http_status_code, 'http_status_type': http_status_type, 'http_status_desc': http_status_desc, 'response_headers': safe_dict(response_headers), } self.logger.debug( '{}: Response: Details'.format(request_label), extra=response_extra, ) http_status_successful = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL ) http_status_redirection = is_http_status_type( http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION ) if http_status_successful or http_status_redirection: if hasattr(response, 'url') and \ response.url and \ len(response.url) > 0: response_extra.update({'response_url': response.url}) self.logger.debug( '{}: Cookie Payload'.format(request_label), extra={'cookie_payload': self.tune_request.session.cookies.get_dict(), 'request_label': request_label} ) assert response return response else: response_extra.update({'error_request_curl': self.built_request_curl}) self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra) json_response_error = \ build_response_error_details( response=response, request_label=request_label, request_url=request_url ) extra_error = copy.deepcopy(json_response_error) if self.logger_level == logging.INFO: error_response_details = \ extra_error.get('response_details', None) if error_response_details and \ isinstance(error_response_details, str) and \ len(error_response_details) > 100: extra_error['response_details'] = error_response_details[:100] + ' ...' if self.built_request_curl and \ 'error_request_curl' not in extra_error: extra_error.update({'error_request_curl': self.built_request_curl}) self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error) kwargs = { 'error_status': json_response_error.get("response_status", None), 'error_reason': json_response_error.get("response_reason", None), 'error_details': json_response_error.get("response_details", None), 'error_request_curl': self.built_request_curl } if http_status_code in [ HttpStatusCode.BAD_REQUEST, HttpStatusCode.UNAUTHORIZED, HttpStatusCode.FORBIDDEN, HttpStatusCode.NOT_FOUND, HttpStatusCode.METHOD_NOT_ALLOWED, HttpStatusCode.NOT_ACCEPTABLE, HttpStatusCode.REQUEST_TIMEOUT, HttpStatusCode.CONFLICT, HttpStatusCode.GONE, HttpStatusCode.UNPROCESSABLE_ENTITY, HttpStatusCode.TOO_MANY_REQUESTS, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestClientError(**kwargs) if http_status_code in [ HttpStatusCode.INTERNAL_SERVER_ERROR, HttpStatusCode.NOT_IMPLEMENTED, HttpStatusCode.BAD_GATEWAY, HttpStatusCode.SERVICE_UNAVAILABLE, HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED, ]: kwargs.update({'error_code': http_status_code}) raise TuneRequestServiceError(**kwargs) kwargs.update({'error_code': json_response_error['response_status_code']}) extra_unhandled = copy.deepcopy(kwargs) extra_unhandled.update({'http_status_code': http_status_code}) self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled) raise TuneRequestModuleError(**kwargs)
def download_csv_transform_to_json(self, response, tmp_directory, tmp_json_file_name, config_job): """Download CSV and Transform to JSON Args: response: tmp_directory: tmp_csv_file_name: request_label: Returns: """ self.logger.debug("Download CSV Transform JSON: Start") if not os.path.exists(tmp_directory): os.mkdir(tmp_directory) tmp_json_file_path = \ "{tmp_directory}/{tmp_json_file_name}".format( tmp_directory=tmp_directory, tmp_json_file_name=tmp_json_file_name ) if os.path.exists(tmp_json_file_path): self.logger.debug("Removing previous JSON File", extra={'file_path': tmp_json_file_path}) os.remove(tmp_json_file_path) line_count = 0 csv_keys_str = None csv_keys_list = None csv_keys_list_len = None pre_str_line = None try: with open(file=tmp_json_file_path, mode='w') as dw_file_w: for bytes_line in response.iter_lines(chunk_size=4096): if bytes_line: # filter out keep-alive new chunks line_count += 1 str_line = bytes_line.decode("utf-8") if line_count == 1: csv_keys_str = str_line csv_keys_list = csv_keys_str.split(',') csv_keys_list_len = len(csv_keys_list) continue elif line_count > 2: dw_file_w.write('\n') if pre_str_line is not None: str_line = pre_str_line + str_line pre_str_line = None csv_values_str = str_line.replace('\n', ' ').replace( '\r', ' ') data = io.StringIO(csv_values_str) reader = csv.reader(data, delimiter=',') csv_values_list = None for row in reader: csv_values_list = row csv_values_list_len = len(csv_values_list) if csv_values_list_len < csv_keys_list_len: pre_str_line = str_line continue if csv_keys_list_len != csv_values_list_len: self.logger.error("Mismatch: CSV Key", extra={ 'line': line_count, 'csv_keys_list_len': csv_keys_list_len, 'csv_keys_str': csv_keys_str, 'csv_keys_list': csv_keys_list, 'csv_values_list_len': csv_values_list_len, 'csv_values_str': csv_values_str, 'csv_values_list': csv_values_list, }) raise TuneRequestModuleError( error_message= "Mismatch: CSV Key '{}': Values '{}'".format( csv_keys_str, csv_values_str), exit_code=TuneIntegrationExitCode. MOD_ERR_UNEXPECTED_VALUE) json_dict = {} for idx, csv_key in enumerate(csv_keys_list): csv_value = csv_values_list[idx] json_dict.update({csv_key: csv_value.strip('"')}) csv_row_mapped = self.map_data_row( data_row=json_dict, config_job=config_job) json_str = json.dumps(csv_row_mapped) dw_file_w.write(json_str) dw_file_w.flush() except requests.exceptions.StreamConsumedError as request_ex: self.logger.error( "Download CSV Transform JSON: Stream Previously Consumed Exception", extra={ 'error_exception': base_class_name(request_ex), 'error_details': get_exception_message(request_ex) }) raise except requests.exceptions.RequestException as request_ex: self.logger.error("Download CSV Transform JSON: Request Exception", extra={ 'error_exception': base_class_name(request_ex), 'error_details': get_exception_message(request_ex) }) raise except Exception as ex: self.logger.error( "Download CSV Transform JSON: Unexpected Exception", extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) }) raise tmp_json_file_size = \ os.path.getsize(tmp_json_file_path) return (tmp_json_file_path, tmp_json_file_size, line_count)
def request_upload_data(self, upload_request_url, upload_data, upload_data_size, request_label=None, upload_timeout=None, build_request_curl=False): """Upload Data to requested URL. :param upload_request_url: :param upload_data: :param upload_data_size: :param upload_timeout: :return: """ _request_label = 'Request Upload Data' request_label = f'{request_label}: {request_label}' if request_label is not None else _request_label log.info(f'{request_label}: Start', extra={ 'upload_data_size': upload_data_size, 'upload_request_url': upload_request_url, }) request_retry_excps = REQUEST_RETRY_EXCPS request_retry_http_status_codes = REQUEST_RETRY_HTTP_STATUS_CODES upload_request_retry = {"timeout": 60, "tries": -1, "delay": 60} request_headers = { 'Content-type': 'application/json; charset=utf8', 'Accept': 'text/plain', 'Content-Length': str(upload_data_size) } if upload_timeout: upload_request_retry["timeout"] = int(upload_timeout) try: response = self.mv_request.request( request_method='PUT', request_url=upload_request_url, request_params=None, request_data=upload_data, request_retry=upload_request_retry, request_retry_excps=request_retry_excps, request_retry_http_status_codes=request_retry_http_status_codes, request_retry_excps_func=mv_request_retry_excps_func, request_headers=request_headers, allow_redirects=False, build_request_curl=build_request_curl, request_label=request_label) except TuneRequestBaseError as tmv_ex: tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({'error_exception': base_class_name(tmv_ex)}) log.error(f'{request_label}: Failed', extra=tmv_ex_extra) raise except Exception as ex: print_traceback(ex) log.error(f'{request_label}: Failed: Unexpected', extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) }) raise TuneRequestModuleError( error_message= f'{request_label}: Failed: {get_exception_message(ex)}', errors=ex, error_code=TuneRequestErrorCodes.REQ_ERR_UPLOAD_DATA) log.info(f'{request_label}: Finished') return response
def _request_retry( self, call_func, fargs=None, fkwargs=None, timeout=60, request_retry_func=None, request_retry_excps_func=None, request_label=None, ): """Request Retry Args: call_func: the function to execute. request_retry_excps: A tuple of exceptions to catch. fargs: the positional arguments of the function to execute. fkwargs: the named arguments of the function to execute. timeout: (optional) How long to wait for the server to send data before giving up. request_retry_func: (optional) Retry alternative to request_retry_excps. retry_tries: the maximum number of attempts. default: -1 (infinite). retry_delay: initial delay between attempts. default: 0. retry_max_delay:the maximum value of delay. default: None (no limit). retry_backoff: multiplier applied to delay between attempts. default: 1 (no backoff). retry_jitter: extra seconds added to delay between attempts. default: 0. request_label: Label Returns: """ request_retry_extra = {'timeout': timeout} request_retry_extra.update({ 'request_retry_http_status_codes': self.request_retry_http_status_codes }) if self.request_retry_excps is not None: request_retry_excp_names = [ excp.__name__ for excp in list(self.request_retry_excps) ] request_retry_extra.update( {'request_retry_excps': request_retry_excp_names}) if request_retry_func is not None: request_retry_func_name = request_retry_func.__name__ request_retry_extra.update( {'request_retry_func': request_retry_func_name}) if request_retry_excps_func is not None: request_retry_excps_func_name = request_retry_excps_func.__name__ request_retry_extra.update( {'request_retry_excps_func': request_retry_excps_func_name}) self.logger.debug("Request Retry: Start: {}".format( request_label if request_label else ""), extra=request_retry_extra) args = fargs if fargs else list() kwargs = fkwargs if fkwargs else dict() request_url = kwargs[ 'request_url'] if kwargs and 'request_url' in kwargs else "" _attempts = 0 _tries, _delay, _timeout = self.retry_tries, self.retry_delay, self.timeout while _tries: _attempts += 1 fkwargs['timeout'] = _timeout request_func = partial(call_func, *args, **kwargs) self.logger.debug("Request Retry: Attempt: {}: {}".format( request_label if request_label else "", _attempts), extra={ 'attempts': _attempts, 'timeout': _timeout, 'tries': _tries, 'delay': _delay, 'request_url': request_url }) error_exception = None _tries -= 1 try: response = request_func() if response is None: raise TuneRequestModuleError( error_message="Request Retry: No response", error_code=TuneRequestErrorCodes. REQ_ERR_UNEXPECTED_VALUE) self.logger.debug("Request Retry: Checking Response", extra={ 'request_url': request_url, 'request_label': request_label }) if request_retry_func is not None: if not request_retry_func(response): self.logger.debug( "Request Retry: Response: Valid: Not Retry Candidate", extra={ 'request_url': request_url, 'request_label': request_label }) return response else: self.logger.debug("Request Retry: Response: Valid", extra={ 'request_url': request_url, 'request_label': request_label }) return response self.logger.debug( "Request Retry: Response: Valid: Retry Candidate", extra={ 'request_url': request_url, 'request_label': request_label }) except self.request_retry_excps as retry_ex: error_exception = retry_ex self.logger.warning( "Request Retry: Expected: {}: Retry Candidate".format( base_class_name(error_exception)), extra={ 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label }) if not _tries: self.logger.error( "Request Retry: Expected: {}: Exhausted Retries". format(base_class_name(error_exception))) raise except TuneRequestBaseError as tmv_ex: error_exception = tmv_ex tmv_ex_extra = tmv_ex.to_dict() tmv_ex_extra.update({ 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_label': request_label }) if not self.request_retry_excps_func or \ not self.request_retry_excps_func(tmv_ex, request_label): self.logger.error( "Request Retry: Integration: {}: Not Retry Candidate". format(base_class_name(error_exception)), extra=tmv_ex_extra) raise self.logger.warning( "Request Retry: Integration: {}: Retry Candidate".format( base_class_name(error_exception)), extra=tmv_ex_extra) if not _tries: self.logger.error( "Request Retry: Integration: {}: Exhausted Retries". format(base_class_name(error_exception))) raise except Exception as ex: error_exception = ex ex_extra = { 'error_exception': base_class_name(error_exception), 'error_details': get_exception_message(error_exception), 'request_url': request_url, 'request_label': request_label } if not self.request_retry_excps_func or \ not self.request_retry_excps_func(error_exception, request_label): self.logger.error( "Request Retry: Unexpected: {}: Not Retry Candidate". format(base_class_name(error_exception)), extra=ex_extra) raise self.logger.warning( "Request Retry: Unexpected: {}: Retry Candidate".format( base_class_name(error_exception)), extra=ex_extra) if not _tries: raise TuneRequestModuleError( error_message="Unexpected: {}".format( base_class_name(error_exception)), errors=error_exception, error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_RETRY_EXHAUSTED) if not _tries: self.logger.error("Request Retry: Exhausted Retries", extra={ 'attempts': _attempts, 'tries': _tries, 'request_url': request_url, 'request_label': request_label }) raise TuneRequestModuleError( error_message=( "Request Retry: Exhausted Retries: {}: {}").format( request_label, request_url), error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED) self.logger.info("Request Retry: Performing Retry", extra={ 'tries': _tries, 'delay': _delay, 'timeout': _timeout, 'request_url': request_url, 'request_label': request_label }) time.sleep(_delay) if self.retry_backoff and self.retry_backoff > 0: _delay *= self.retry_backoff if self.retry_jitter and self.retry_jitter > 0: _delay += self.retry_jitter if self.retry_max_delay is not None: _delay = min(_delay, self.retry_max_delay)
def request_json_download( self, request_method, request_url, tmp_json_file_name, tmp_directory, request_params=None, request_data=None, request_retry=None, request_retry_func=None, request_retry_excps=None, request_retry_excps_func=None, request_headers=None, request_auth=None, request_label=None, build_request_curl=False, allow_redirects=True, verify=True, encoding_write=None, encoding_read=None, ): """Download and Read JSON file. Args: request_method: request_method for the new :class:`Request` object. request_url: URL for the new :class:`Request` object. tmp_json_file_name: Provide temporary name for downloaded CSV tmp_directory: Provide temporary directory to hold downloaded CSV request_params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. request_data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. request_retry: (optional) Retry configuration. request_headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. build_request_curl: (optional) Build a copy-n-paste curl for command line that provides same request as this call. allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. encoding_write: encoding_read: decode_unicode: Returns: Generator containing JSON data by rows in JSON dictionary format. """ _request_label = 'Request Download JSON File' request_label = f'{request_label}: {_request_label}' if request_label is not None else _request_label log.info(f'{request_label}: Start', extra={ 'request_url': request_url, 'encoding_write': encoding_write, 'encoding_read': encoding_read, }) timer_start = dt.datetime.now() _attempts = 0 _tries = 60 _delay = 10 while _tries: _attempts += 1 log.debug(f'{request_label}: Download', extra={ 'attempts': _attempts, 'request_url': request_url, }) response = self.mv_request.request( request_method=request_method, request_url=request_url, request_params=request_params, request_data=request_data, request_retry=request_retry, request_retry_func=request_retry_func, request_retry_excps=request_retry_excps, request_retry_excps_func=request_retry_excps_func, request_headers=request_headers, request_auth=request_auth, build_request_curl=build_request_curl, allow_redirects=allow_redirects, verify=verify, stream=True, request_label=request_label) if response is None: log.error(f'{request_label}: No response', extra={ 'request_url': request_url, }) raise TuneRequestModuleError( error_message=f'{request_label}: No response', error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST) http_status_code = response.status_code timer_end = dt.datetime.now() timer_delta = timer_end - timer_start response_time_secs = timer_delta.seconds response_headers = None if hasattr(response, 'headers'): response_headers = \ json.loads( json.dumps( dict(response.headers) ) ) log.debug(f'{request_label}: Response Status', extra={ 'http_status_code': http_status_code, 'response_time_secs': response_time_secs, 'response_url': response.url, 'response_headers': safe_dict(response_headers), }) if not os.path.exists(tmp_directory): os.mkdir(tmp_directory) tmp_json_file_path = f'{tmp_directory}/{tmp_json_file_name}' if os.path.exists(tmp_json_file_path): log.debug( f'{request_label}: Removing', extra={'file_path': tmp_json_file_path}, ) os.remove(tmp_json_file_path) mode_write = 'wb' if encoding_write is None else 'w' log.debug(f'{request_label}: Finished', extra={ 'file_path': tmp_json_file_path, 'mode_write': mode_write, 'encoding_write': encoding_write, }) log.debug(f'{request_label}: Usage', extra=env_usage(tmp_directory)) chunk_total_sum = 0 with open(file=tmp_json_file_path, mode=mode_write, encoding=encoding_write) as json_raw_file_w: log.debug(f'{request_label}: Response Raw: Started', extra={ 'file_path': tmp_json_file_path, }) _tries -= 1 error_exception = None error_details = None chunk_size = 8192 try: raw_response = response.raw while True: chunk = raw_response.read(chunk_size, decode_content=True) if not chunk: break chunk_total_sum += chunk_size json_raw_file_w.write(chunk) json_raw_file_w.flush() os.fsync(json_raw_file_w.fileno()) log.debug(f'{request_label}: By Chunk: Completed', extra={ 'file_path': tmp_json_file_path, }) break except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex: error_exception = base_class_name(chunked_encoding_ex) error_details = get_exception_message(chunked_encoding_ex) log.warning(f'{request_label}: Error: {error_exception}', extra={ 'error_details': error_details, 'chunk_total_sum': chunk_total_sum, }) if not _tries: log.error( f'{request_label}: Exhausted Retries: Error: {error_exception}' ) raise except http_client.IncompleteRead as incomplete_read_ex: error_exception = base_class_name(incomplete_read_ex) error_details = get_exception_message(incomplete_read_ex) log.warning(f'{request_label}: IncompleteRead', extra={ 'error_exception': error_exception, 'error_details': error_details, 'chunk_total_sum': chunk_total_sum, }) if not _tries: log.error( f'{request_label}: Exhausted Retries: Error: {error_exception}' ) raise except requests.exceptions.RequestException as request_ex: log.error(f'{request_label}: Request Exception', extra={ 'error_exception': base_class_name(request_ex), 'error_details': get_exception_message(request_ex), 'chunk_total_sum': chunk_total_sum, }) raise except Exception as ex: log.error(f'{request_label}: Unexpected Exception', extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex), 'chunk_total_sum': chunk_total_sum, }) raise if not _tries: log.error(f'{request_label}: Exhausted Retries', extra={ 'tries': _tries, 'request_url': request_url, }) raise TuneRequestModuleError( error_message= f'{request_label}: Exhausted Retries: {request_url}', error_request_curl=self.built_request_curl, error_code=TuneRequestErrorCodes. REQ_ERR_RETRY_EXHAUSTED) log.info(f'{request_label}: Performing Retry', extra={ 'tries': _tries, 'delay': _delay, 'request_url': request_url, }) time.sleep(_delay) tmp_json_file_size = os.path.getsize(tmp_json_file_path) bom_enc, bom_len, bom_header = detect_bom(tmp_json_file_path) log.info(f'{request_label}: By Chunk: Completed: Details', extra={ 'file_path': tmp_json_file_path, 'file_size': bytes_to_human(tmp_json_file_size), 'chunk_total_sum': chunk_total_sum, 'bom_encoding': bom_enc, }) if bom_enc == 'gzip': tmp_json_gz_file_path = f"{tmp_json_file_path}.gz" os.rename(src=tmp_json_file_path, dst=tmp_json_gz_file_path) with open(file=tmp_json_file_path, mode=mode_write, encoding=encoding_write) as json_file_w: log.debug(f'{request_label}: GZip: Started', extra={ 'file_path': tmp_json_file_path, }) with gzip.open(tmp_json_gz_file_path, 'r') as gzip_file_r: json_file_w.write(gzip_file_r.read()) response_extra = { 'file_path': tmp_json_file_path, 'file_size': bytes_to_human(tmp_json_file_size), } log.info(f'{request_label}: Read Downloaded', extra=response_extra) json_download = None with open(tmp_json_file_path, mode='r') as json_file_r: json_file_content = json_file_r.read() try: json_download = json.loads(json_file_content) except ValueError as json_decode_ex: pprint(json_file_content) response_extra.update({ 'json_file_content': json_file_content, 'json_file_content_len': len(json_file_content) }) handle_json_decode_error(response_decode_ex=json_decode_ex, response=response, response_extra=response_extra, request_label=request_label, request_curl=self.built_request_curl) except Exception as ex: pprint(json_file_content) response_extra.update({ 'json_file_content': json_file_content, 'json_file_content_len': len(json_file_content) }) log.error( f'{request_label}: Failed: Exception', extra=response_extra, ) handle_json_decode_error(response_decode_ex=ex, response=response, response_extra=response_extra, request_label=request_label, request_curl=self.built_request_curl) response_extra.update({'json_file_content_len': len(json_download)}) log.info(f'{request_label}: Finished', extra=response_extra) return json_download
def download_csv( self, response, tmp_directory, tmp_csv_file_name, request_label=None, encoding_write=None, decode_unicode=False, ): _request_label = 'Download CSV' request_label = f'{request_label}: {_request_label}' if request_label is not None else _request_label log.debug(f'{request_label}: Start') if not os.path.exists(tmp_directory): os.mkdir(tmp_directory) tmp_csv_file_path = \ f'{tmp_directory}/{tmp_csv_file_name}' if os.path.exists(tmp_csv_file_path): log.debug( f'{request_label}: Removing previous CSV', extra={'file_path': tmp_csv_file_path}, ) os.remove(tmp_csv_file_path) mode_write = 'wb' if encoding_write is None else 'w' log.debug(f'{request_label}: Details', extra={ 'file_path': tmp_csv_file_path, 'mode_write': mode_write, 'encoding_write': encoding_write, }) chunk_total_sum = 0 with open(file=tmp_csv_file_path, mode=mode_write, encoding=encoding_write) as csv_file_wb: log.debug(f'{request_label}: By Chunk: Started', extra={ 'file_path': tmp_csv_file_path, 'request_label': request_label }) error_exception = None error_details = None try: for chunk in response.iter_content( chunk_size=8192, decode_unicode=decode_unicode): if not chunk: break chunk_total_sum += 8192 csv_file_wb.write(chunk) csv_file_wb.flush() os.fsync(csv_file_wb.fileno()) log.debug(f'{request_label}: By Chunk: Completed', extra={ 'file_path': tmp_csv_file_path, }) except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex: error_exception = base_class_name(chunked_encoding_ex) error_details = get_exception_message(chunked_encoding_ex) log.warning(f'{request_label}: ChunkedEncodingError', extra={ 'error_exception': error_exception, 'error_details': error_details, 'chunk_total_sum': bytes_to_human(chunk_total_sum), }) return (None, 0) except http_client.IncompleteRead as incomplete_read_ex: error_exception = base_class_name(incomplete_read_ex) error_details = get_exception_message(incomplete_read_ex) log.warning(f'{request_label}: IncompleteRead', extra={ 'error_exception': error_exception, 'error_details': error_details, 'chunk_total_sum': bytes_to_human(chunk_total_sum), }) return (None, 0) except requests.exceptions.RequestException as request_ex: log.error(f'{request_label}: Request Exception', extra={ 'error_exception': base_class_name(request_ex), 'error_details': get_exception_message(request_ex), 'chunk_total_sum': bytes_to_human(chunk_total_sum), }) raise except Exception as ex: log.error(f'{request_label}: Unexpected Exception', extra={ 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex), 'chunk_total_sum': bytes_to_human(chunk_total_sum), }) raise tmp_csv_file_size = os.path.getsize(tmp_csv_file_path) bom_enc, bom_len, bom_header = detect_bom(tmp_csv_file_path) log.info(f'{request_label}: By Chunk: Completed: Details', extra={ 'file_path': tmp_csv_file_path, 'file_size': bytes_to_human(tmp_csv_file_size), 'chunk_total_sum': bytes_to_human(chunk_total_sum), 'bom_encoding': bom_enc }) log.debug("Download CSV: Usage", extra=env_usage(tmp_directory)) tmp_csv_file_name_wo_ext = \ os.path.splitext( os.path.basename(tmp_csv_file_name) )[0] tmp_csv_file_path_wo_bom = \ f'{tmp_directory}/{tmp_csv_file_name_wo_ext}_wo_bom.csv' if os.path.exists(tmp_csv_file_path_wo_bom): os.remove(tmp_csv_file_path_wo_bom) bom_enc, bom_len = remove_bom(tmp_csv_file_path, tmp_csv_file_path_wo_bom) log.debug(f'{request_label}: Encoding', extra={ 'bom_enc': bom_enc, 'bom_len': bom_len }) if bom_len > 0: tmp_csv_file_path = tmp_csv_file_path_wo_bom return (tmp_csv_file_path, tmp_csv_file_size)
def handle_json_decode_error( response_decode_ex, response, response_extra=None, request_label=None, request_curl=None, ): """Handle JSON Decode Error Args: response_json_decode_error: response: response_extra: request_label: Returns: """ if response_extra is None: response_extra = {} if request_label: response_extra.update({'request_label': request_label}) if hasattr(response, 'text') and \ response.text and \ len(response.text) > 0: response_details = response.text response_details_source = 'text' response_content_length = len(response_details) if response_details.startswith('<html'): response_details_source = 'html' soup_html = bs4.BeautifulSoup(response_details, "html.parser") # kill all script and style elements for script in soup_html(["script", "style"]): script.extract() # rip it out text_html = soup_html.get_text() lines_html = [ line for line in text_html.split('\n') if line.strip() != '' ] lines_html = [line.strip(' ') for line in lines_html] response_details = lines_html elif response_details.startswith('<?xml'): response_details_source = 'xml' response_details = json.dumps(xmltodict.parse(response_details)) else: pprint(response_details) response_extra.update({ 'response_details': response_details, 'response_details_source': response_details_source, 'response_content_length': response_content_length, 'error_exception': base_class_name(response_decode_ex), 'error_details': get_exception_message(response_decode_ex) }) log.error("Validate JSON Response: Failed: Invalid", extra=response_extra) raise TuneRequestModuleError( error_message="Validate JSON Response: Failed: Invalid", errors=response_decode_ex, error_request_curl=request_curl, error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)