def is_retry_non_retry_ex(self, _tries, request_label, tmv_ex):
is_retry = True
error_exception = tmv_ex
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_label': request_label
})
self.logger.warning(
"Request Retry: Failed: {}".format(get_exception_message(error_exception)), extra=tmv_ex.to_dict()
)
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(tmv_ex, request_label):
tmv_ex_extra.update({'request_retry_excps_func': self.request_retry_excps_func})
self.logger.error(
"Request Retry: Integration: {}: Not Retry Candidate".format(base_class_name(error_exception)),
extra=tmv_ex_extra
)
is_retry = False
if is_retry:
self.logger.warning(
"Request Retry: Integration: {}: Retry Candidate".format(base_class_name(error_exception)),
extra=tmv_ex_extra
)
is_retry = not self.is_exhausted_retries(
_tries,
partial(
self.logger.error,
"Request Retry: Expected: {}: Exhausted Retries".format(base_class_name(error_exception))
)
)
return is_retry
def is_retry_retry_ex(self, tries, request_url, retry_ex, request_label=None):
"""Is Retry Retry Exception
:param tries:
:param request_url:
:param retry_ex:
:param request_label:
:return:
"""
_request_label = 'Is Retry Retry Exception'
request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label
self.logger.warning(
'{}: Expected: {}: Retry Candidate'.format(request_label, base_class_name(retry_ex)),
extra={
'error_details': get_exception_message(retry_ex),
'request_url': request_url,
'request_label': request_label
}
)
return not self.is_exhausted_retries(
tries,
partial(
self.logger.error,
'{}: Expected: {}: Exhausted Retries'.format(request_label, base_class_name(retry_ex))
)
)
def is_retry_non_tune_ex(self, _tries, ex, request_label, request_url):
is_retry = True
raised_exception = None
error_exception = ex
ex_extra = {
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
}
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(error_exception, request_label):
self.logger.error(
"Request Retry: Unexpected: {}: Not Retry Candidate".format(base_class_name(error_exception)),
extra=ex_extra
)
is_retry = False
raised_exception = error_exception
if is_retry:
self.logger.warning(
"Request Retry: Unexpected: {}: Retry Candidate".format(base_class_name(error_exception)),
extra=ex_extra
)
if self.is_exhausted_retries(_tries, lambda: None):
is_retry = False
raised_exception = TuneRequestModuleError(
error_message="Unexpected: {}".format(base_class_name(error_exception)),
errors=error_exception,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
return is_retry, raised_exception
def _upload_request_retry_excps_func(self, excp, request_label):
"""Upload Request Retry Exception Function
Args:
excp:
Returns:
"""
error_exception = base_class_name(excp)
error_details = get_exception_message(excp)
if isinstance(excp, TuneRequestBaseError):
log.debug("Request Retry: Upload Exception Func",
extra={
'request_label': request_label,
'error_exception': error_exception,
'error_details': error_details
})
else:
log.debug("Request Retry: Upload Exception Func: Unexpected",
extra={
'request_label': request_label,
'error_exception': error_exception,
'error_details': error_details
})
if isinstance(excp, TuneRequestBaseError) and \
excp.error_code == TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT:
if error_details.find('RemoteDisconnected') >= 0 or \
error_details.find('ConnectionResetError') >= 0:
log.debug("Request Retry: Upload Exception Func: Retry",
extra={
'request_label': request_label,
'error_exception': error_exception,
'error_details': error_details
})
return True
if isinstance(excp, requests.exceptions.ConnectionError):
if error_details.find('RemoteDisconnected') >= 0 or \
error_details.find('ConnectionResetError') >= 0:
log.debug("Request Retry: Upload Exception Func: Retry",
extra={
'request_label': request_label,
'error_exception': error_exception,
'error_details': error_details
})
return True
log.debug("Request Retry: Upload Exception Func: Not Retry",
extra={
'request_label': request_label,
'error_exception': error_exception,
'error_details': error_details
})
return False
def is_retry_non_retry_ex(self, tries, tmv_ex, request_label=None):
"""Is Retry Non-Retry Exception
:param tries:
:param tmv_ex:
:param request_label:
:return:
"""
_request_label = 'Is Retry Non-Retry Exception'
request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label
is_retry = True
error_exception = tmv_ex
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_label': request_label
})
self.logger.warning(
'{}: Failed: {}'.format(request_label, get_exception_message(error_exception)),
extra=tmv_ex.to_dict(),
)
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(tmv_ex, request_label):
tmv_ex_extra.update({'request_retry_excps_func': self.request_retry_excps_func})
self.logger.error(
'{}: Integration: {}: Not Retry Candidate'.format(request_label, base_class_name(error_exception)),
extra=tmv_ex_extra
)
is_retry = False
if is_retry:
self.logger.warning(
'{}: Integration: {}: Retry Candidate'.format(request_label, base_class_name(error_exception)),
extra=tmv_ex_extra
)
is_retry = not self.is_exhausted_retries(
tries,
partial(
self.logger.error,
'{}: Expected: {}: Exhausted Retries'.format(request_label, base_class_name(error_exception))
)
)
return is_retry
def request(self, request_method, request_url, **kwargs):
extra_session_request = {'method': request_method, 'url': request_url}
extra_session_request.update(kwargs)
log.debug("Session Request: Details", extra=extra_session_request)
try:
return self.session.request(method=request_method, url=request_url, **kwargs)
except Exception as ex:
log.warning(
"Session Request: Failed: {}".format(get_exception_message(ex)),
extra=extra_session_request,
)
raise
def is_retry_retry_ex(self, tries, request_label, request_url, retry_ex):
self.logger.warning(
"Request Retry: Expected: {}: Retry Candidate".format(base_class_name(retry_ex)),
extra={
'error_details': get_exception_message(retry_ex),
'request_url': request_url,
'request_label': request_label
}
)
return not self.is_exhausted_retries(
tries,
partial(
self.logger.error, "Request Retry: Expected: {}: Exhausted Retries".format(base_class_name(retry_ex))
)
)
def is_retry_non_tune_ex(self, tries, ex, request_url, request_label=None):
"""Is Retry Non-TUNE Exception
:param tries:
:param ex:
:param request_url:
:param request_label:
:return:
"""
_request_label = 'Is Retry Non-TUNE Exception'
request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label
is_retry = True
raised_exception = None
error_exception = ex
ex_extra = {
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
}
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(error_exception, request_label):
self.logger.error(
'{}: Unexpected: {}: Not Retry Candidate'.format(request_label, base_class_name(error_exception)),
extra=ex_extra
)
is_retry = False
raised_exception = error_exception
if is_retry:
self.logger.warning(
'{}: Unexpected: {}: Retry Candidate'.format(request_label, base_class_name(error_exception)),
extra=ex_extra
)
if self.is_exhausted_retries(tries, lambda: None):
is_retry = False
raised_exception = TuneRequestModuleError(
error_message='{}: Unexpected: {}'.format(request_label, base_class_name(error_exception)),
errors=error_exception,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
return is_retry, raised_exception
def request_safe(self,
request_method,
request_url,
response_hook=None,
exception_handler=None,
**kwargs):
response_hook, exception_handler = self.create_hooks(
response_hook, exception_handler)
try:
return self.session.request(method=request_method,
url=request_url,
hooks={'response': response_hook},
**kwargs)
except Exception as ex:
log.warning("Session Request: Failed: {}".format(
get_exception_message(ex)),
extra={
'request_method': request_method,
'request_url': request_url
})
exception_handler(ex)
return None
def request_upload_json_file(self,
upload_request_url,
upload_data_file_path,
upload_data_file_size,
is_upload_gzip,
request_label,
upload_timeout=None):
"""Upload File to requested URL.
Args:
upload_request_url:
upload_data_file_path:
upload_data_file_size:
upload_timeout:
Returns:
"""
request_retry_excps = REQUEST_RETRY_EXCPS
request_retry_http_status_codes = REQUEST_RETRY_HTTP_STATUS_CODES
upload_request_retry = {"timeout": 60, "tries": -1, "delay": 60}
upload_request_headers = {
'Content-Length': '{}'.format(upload_data_file_size)
}
if is_upload_gzip:
upload_request_headers.update({'Content-Type': 'application/gzip'})
else:
upload_request_headers.update(
{'Content-Type': 'application/json; charset=utf8'})
if upload_timeout:
upload_request_retry["timeout"] = int(upload_timeout)
upload_extra = {
'upload_request_url': upload_request_url,
'upload_data_file_path': upload_data_file_path,
'upload_data_file_size': upload_data_file_size,
'upload_request_retry': upload_request_retry,
'upload_request_headers': upload_request_headers
}
log.info("Request Upload JSON File: Start", extra=upload_extra)
try:
with open(upload_data_file_path, 'rb') as upload_fp:
response = self.mv_request.request(
request_method="PUT",
request_url=upload_request_url,
request_params=None,
request_data=upload_fp,
request_retry=upload_request_retry,
request_headers=upload_request_headers,
request_retry_excps=request_retry_excps,
request_retry_http_status_codes=
request_retry_http_status_codes,
request_retry_excps_func=self.
_upload_request_retry_excps_func,
allow_redirects=False,
build_request_curl=False,
request_label=request_label)
except TuneRequestBaseError as tmv_ex:
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({'error_exception': base_class_name(tmv_ex)})
log.error("Request Upload JSON File: Failed", extra=tmv_ex_extra)
raise
except Exception as ex:
log.error("Request Upload JSON File: Failed: Unexpected",
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
})
print_traceback(ex)
raise TuneRequestModuleError(
error_message=(
"Request Upload JSON File: Failed: Unexpected: {}: {}"
).format(base_class_name(ex), get_exception_message(ex)),
errors=ex,
error_code=TuneRequestErrorCodes.REQ_ERR_UPLOAD_DATA)
log.info("Request Upload JSON File: Finished")
return response
def download_csv(self,
response,
tmp_directory,
tmp_csv_file_name,
request_label=None,
encoding_write=None,
decode_unicode=False):
self.logger.debug("Download CSV: Start")
if not os.path.exists(tmp_directory):
os.mkdir(tmp_directory)
tmp_csv_file_path = \
"{tmp_directory}/{tmp_csv_file_name}".format(
tmp_directory=tmp_directory,
tmp_csv_file_name=tmp_csv_file_name
)
if os.path.exists(tmp_csv_file_path):
self.logger.debug("Removing previous CSV",
extra={'file_path': tmp_csv_file_path})
os.remove(tmp_csv_file_path)
mode_write = 'wb' if encoding_write is None else 'w'
self.logger.debug("Download CSV: Details",
extra={
'file_path': tmp_csv_file_path,
'mode_write': mode_write,
'encoding_write': encoding_write,
'request_label': request_label
})
chunk_total_sum = 0
with open(file=tmp_csv_file_path,
mode=mode_write,
encoding=encoding_write) as csv_file_wb:
self.logger.debug("Download CSV: By Chunk: Started",
extra={
'file_path': tmp_csv_file_path,
'request_label': request_label
})
error_exception = None
error_details = None
try:
for chunk in response.iter_content(
chunk_size=8192, decode_unicode=decode_unicode):
if not chunk:
break
chunk_total_sum += 8192
csv_file_wb.write(chunk)
csv_file_wb.flush()
os.fsync(csv_file_wb.fileno())
self.logger.debug("Download CSV: By Chunk: Completed",
extra={
'file_path': tmp_csv_file_path,
'request_label': request_label
})
except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex:
error_exception = base_class_name(chunked_encoding_ex)
error_details = get_exception_message(chunked_encoding_ex)
self.logger.warning("Download CSV: ChunkedEncodingError",
extra={
'error_exception':
error_exception,
'error_details':
error_details,
'chunk_total_sum':
convert_size(chunk_total_sum),
'request_label':
request_label
})
return (None, 0)
except http_client.IncompleteRead as incomplete_read_ex:
error_exception = base_class_name(incomplete_read_ex)
error_details = get_exception_message(incomplete_read_ex)
self.logger.warning("Download CSV: IncompleteRead",
extra={
'error_exception':
error_exception,
'error_details':
error_details,
'chunk_total_sum':
convert_size(chunk_total_sum),
'request_label':
request_label
})
return (None, 0)
except requests.exceptions.RequestException as request_ex:
self.logger.error("Download CSV: Request Exception",
extra={
'error_exception':
base_class_name(request_ex),
'error_details':
get_exception_message(request_ex),
'chunk_total_sum':
convert_size(chunk_total_sum),
'request_label':
request_label
})
raise
except Exception as ex:
self.logger.error("Download CSV: Unexpected Exception",
extra={
'error_exception':
base_class_name(ex),
'error_details':
get_exception_message(ex),
'chunk_total_sum':
convert_size(chunk_total_sum),
'request_label':
request_label
})
raise
tmp_csv_file_size = os.path.getsize(tmp_csv_file_path)
bom_enc, bom_len, bom_header = detect_bom(tmp_csv_file_path)
self.logger.debug("Download CSV: By Chunk: Completed: Details",
extra={
'file_path': tmp_csv_file_path,
'file_size': convert_size(tmp_csv_file_size),
'chunk_total_sum': convert_size(chunk_total_sum),
'bom_encoding': bom_enc
})
tmp_csv_file_name_wo_ext = \
os.path.splitext(
os.path.basename(tmp_csv_file_name)
)[0]
tmp_csv_file_path_wo_bom = \
"{tmp_directory}/{tmp_csv_file_name}_wo_bom.csv".format(
tmp_directory=tmp_directory,
tmp_csv_file_name=tmp_csv_file_name_wo_ext
)
if os.path.exists(tmp_csv_file_path_wo_bom):
os.remove(tmp_csv_file_path_wo_bom)
bom_enc, bom_len = remove_bom(tmp_csv_file_path,
tmp_csv_file_path_wo_bom)
self.logger.debug("Download CSV: Encoding",
extra={
'bom_enc': bom_enc,
'bom_len': bom_len
})
if bom_len > 0:
tmp_csv_file_path = tmp_csv_file_path_wo_bom
return (tmp_csv_file_path, tmp_csv_file_size)
def request_upload_data(self,
upload_request_url,
upload_data,
upload_data_size,
upload_timeout=None):
"""Upload Data to requested URL.
Args:
upload_request_url:
upload_data:
Returns:
requests.Response
"""
log.info("Request Upload JSON Data: Start",
extra={
'upload_data_size': upload_data_size,
'upload_request_url': upload_request_url
})
request_retry_excps = REQUEST_RETRY_EXCPS
request_retry_http_status_codes = REQUEST_RETRY_HTTP_STATUS_CODES
upload_request_retry = {"timeout": 60, "tries": -1, "delay": 60}
request_headers = {
'Content-type': 'application/json; charset=utf8',
'Accept': 'text/plain',
'Content-Length': "{}".format(upload_data_size)
}
if upload_timeout:
upload_request_retry["timeout"] = int(upload_timeout)
try:
response = self.mv_request.request(
request_method="PUT",
request_url=upload_request_url,
request_params=None,
request_data=upload_data,
request_retry=upload_request_retry,
request_retry_excps=request_retry_excps,
request_retry_http_status_codes=request_retry_http_status_codes,
request_retry_excps_func=self._upload_request_retry_excps_func,
request_headers=request_headers,
allow_redirects=False,
build_request_curl=False,
request_label="Upload Data to URL")
except TuneRequestBaseError as tmv_ex:
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({'error_exception': base_class_name(tmv_ex)})
log.error("Upload: Failed", extra=tmv_ex_extra)
raise
except Exception as ex:
print_traceback(ex)
log.error("Upload: Failed: Unexpected",
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
})
raise TuneRequestModuleError(
error_message=("RequestMvIntegration: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneRequestErrorCodes.REQ_ERR_UPLOAD_DATA)
log.info("Request Upload JSON Data: Finished")
return response
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise TuneRequestValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise TuneRequestValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
'{}: Session: Details'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug('{}: Details'.format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.debug(
'{}: Curl'.format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url})
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.tune_request.request(**kwargs)
except Exception as ex:
self.logger.error(
'{}: Request Base: Error'.format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
'{}: Response: Failed'.format(request_label),
extra={'request_curl': self.built_request_curl},
)
raise TuneRequestModuleError(
error_message='{}: Response: Failed'.format(request_label),
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
http_status_code_to_type(http_status_code)
http_status_desc = \
http_status_code_to_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
'{}: Response: Details'.format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
'{}: Cookie Payload'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled)
raise TuneRequestModuleError(**kwargs)
def download_csv_transform_to_json(self, response, tmp_directory,
tmp_json_file_name, config_job):
"""Download CSV and Transform to JSON
Args:
response:
tmp_directory:
tmp_csv_file_name:
request_label:
Returns:
"""
self.logger.debug("Download CSV Transform JSON: Start")
if not os.path.exists(tmp_directory):
os.mkdir(tmp_directory)
tmp_json_file_path = \
"{tmp_directory}/{tmp_json_file_name}".format(
tmp_directory=tmp_directory,
tmp_json_file_name=tmp_json_file_name
)
if os.path.exists(tmp_json_file_path):
self.logger.debug("Removing previous JSON File",
extra={'file_path': tmp_json_file_path})
os.remove(tmp_json_file_path)
line_count = 0
csv_keys_str = None
csv_keys_list = None
csv_keys_list_len = None
pre_str_line = None
try:
with open(file=tmp_json_file_path, mode='w') as dw_file_w:
for bytes_line in response.iter_lines(chunk_size=4096):
if bytes_line: # filter out keep-alive new chunks
line_count += 1
str_line = bytes_line.decode("utf-8")
if line_count == 1:
csv_keys_str = str_line
csv_keys_list = csv_keys_str.split(',')
csv_keys_list_len = len(csv_keys_list)
continue
elif line_count > 2:
dw_file_w.write('\n')
if pre_str_line is not None:
str_line = pre_str_line + str_line
pre_str_line = None
csv_values_str = str_line.replace('\n', ' ').replace(
'\r', ' ')
data = io.StringIO(csv_values_str)
reader = csv.reader(data, delimiter=',')
csv_values_list = None
for row in reader:
csv_values_list = row
csv_values_list_len = len(csv_values_list)
if csv_values_list_len < csv_keys_list_len:
pre_str_line = str_line
continue
if csv_keys_list_len != csv_values_list_len:
self.logger.error("Mismatch: CSV Key",
extra={
'line':
line_count,
'csv_keys_list_len':
csv_keys_list_len,
'csv_keys_str':
csv_keys_str,
'csv_keys_list':
csv_keys_list,
'csv_values_list_len':
csv_values_list_len,
'csv_values_str':
csv_values_str,
'csv_values_list':
csv_values_list,
})
raise TuneRequestModuleError(
error_message=
"Mismatch: CSV Key '{}': Values '{}'".format(
csv_keys_str, csv_values_str),
exit_code=TuneIntegrationExitCode.
MOD_ERR_UNEXPECTED_VALUE)
json_dict = {}
for idx, csv_key in enumerate(csv_keys_list):
csv_value = csv_values_list[idx]
json_dict.update({csv_key: csv_value.strip('"')})
csv_row_mapped = self.map_data_row(
data_row=json_dict, config_job=config_job)
json_str = json.dumps(csv_row_mapped)
dw_file_w.write(json_str)
dw_file_w.flush()
except requests.exceptions.StreamConsumedError as request_ex:
self.logger.error(
"Download CSV Transform JSON: Stream Previously Consumed Exception",
extra={
'error_exception': base_class_name(request_ex),
'error_details': get_exception_message(request_ex)
})
raise
except requests.exceptions.RequestException as request_ex:
self.logger.error("Download CSV Transform JSON: Request Exception",
extra={
'error_exception':
base_class_name(request_ex),
'error_details':
get_exception_message(request_ex)
})
raise
except Exception as ex:
self.logger.error(
"Download CSV Transform JSON: Unexpected Exception",
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
})
raise
tmp_json_file_size = \
os.path.getsize(tmp_json_file_path)
return (tmp_json_file_path, tmp_json_file_size, line_count)
def request_upload_data(self,
upload_request_url,
upload_data,
upload_data_size,
request_label=None,
upload_timeout=None,
build_request_curl=False):
"""Upload Data to requested URL.
:param upload_request_url:
:param upload_data:
:param upload_data_size:
:param upload_timeout:
:return:
"""
_request_label = 'Request Upload Data'
request_label = f'{request_label}: {request_label}' if request_label is not None else _request_label
log.info(f'{request_label}: Start',
extra={
'upload_data_size': upload_data_size,
'upload_request_url': upload_request_url,
})
request_retry_excps = REQUEST_RETRY_EXCPS
request_retry_http_status_codes = REQUEST_RETRY_HTTP_STATUS_CODES
upload_request_retry = {"timeout": 60, "tries": -1, "delay": 60}
request_headers = {
'Content-type': 'application/json; charset=utf8',
'Accept': 'text/plain',
'Content-Length': str(upload_data_size)
}
if upload_timeout:
upload_request_retry["timeout"] = int(upload_timeout)
try:
response = self.mv_request.request(
request_method='PUT',
request_url=upload_request_url,
request_params=None,
request_data=upload_data,
request_retry=upload_request_retry,
request_retry_excps=request_retry_excps,
request_retry_http_status_codes=request_retry_http_status_codes,
request_retry_excps_func=mv_request_retry_excps_func,
request_headers=request_headers,
allow_redirects=False,
build_request_curl=build_request_curl,
request_label=request_label)
except TuneRequestBaseError as tmv_ex:
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({'error_exception': base_class_name(tmv_ex)})
log.error(f'{request_label}: Failed', extra=tmv_ex_extra)
raise
except Exception as ex:
print_traceback(ex)
log.error(f'{request_label}: Failed: Unexpected',
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
})
raise TuneRequestModuleError(
error_message=
f'{request_label}: Failed: {get_exception_message(ex)}',
errors=ex,
error_code=TuneRequestErrorCodes.REQ_ERR_UPLOAD_DATA)
log.info(f'{request_label}: Finished')
return response
def _request_retry(
self,
call_func,
fargs=None,
fkwargs=None,
timeout=60,
request_retry_func=None,
request_retry_excps_func=None,
request_label=None,
):
"""Request Retry
Args:
call_func: the function to execute.
request_retry_excps: A tuple of exceptions to catch.
fargs: the positional arguments of the function to execute.
fkwargs: the named arguments of the function to execute.
timeout: (optional) How long to wait for the server to send
data before giving up.
request_retry_func: (optional) Retry alternative to request_retry_excps.
retry_tries: the maximum number of attempts.
default: -1 (infinite).
retry_delay: initial delay between attempts.
default: 0.
retry_max_delay:the maximum value of delay.
default: None (no limit).
retry_backoff: multiplier applied to delay between attempts.
default: 1 (no backoff).
retry_jitter: extra seconds added to delay between attempts.
default: 0.
request_label: Label
Returns:
"""
request_retry_extra = {'timeout': timeout}
request_retry_extra.update({
'request_retry_http_status_codes':
self.request_retry_http_status_codes
})
if self.request_retry_excps is not None:
request_retry_excp_names = [
excp.__name__ for excp in list(self.request_retry_excps)
]
request_retry_extra.update(
{'request_retry_excps': request_retry_excp_names})
if request_retry_func is not None:
request_retry_func_name = request_retry_func.__name__
request_retry_extra.update(
{'request_retry_func': request_retry_func_name})
if request_retry_excps_func is not None:
request_retry_excps_func_name = request_retry_excps_func.__name__
request_retry_extra.update(
{'request_retry_excps_func': request_retry_excps_func_name})
self.logger.debug("Request Retry: Start: {}".format(
request_label if request_label else ""),
extra=request_retry_extra)
args = fargs if fargs else list()
kwargs = fkwargs if fkwargs else dict()
request_url = kwargs[
'request_url'] if kwargs and 'request_url' in kwargs else ""
_attempts = 0
_tries, _delay, _timeout = self.retry_tries, self.retry_delay, self.timeout
while _tries:
_attempts += 1
fkwargs['timeout'] = _timeout
request_func = partial(call_func, *args, **kwargs)
self.logger.debug("Request Retry: Attempt: {}: {}".format(
request_label if request_label else "", _attempts),
extra={
'attempts': _attempts,
'timeout': _timeout,
'tries': _tries,
'delay': _delay,
'request_url': request_url
})
error_exception = None
_tries -= 1
try:
response = request_func()
if response is None:
raise TuneRequestModuleError(
error_message="Request Retry: No response",
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_VALUE)
self.logger.debug("Request Retry: Checking Response",
extra={
'request_url': request_url,
'request_label': request_label
})
if request_retry_func is not None:
if not request_retry_func(response):
self.logger.debug(
"Request Retry: Response: Valid: Not Retry Candidate",
extra={
'request_url': request_url,
'request_label': request_label
})
return response
else:
self.logger.debug("Request Retry: Response: Valid",
extra={
'request_url': request_url,
'request_label': request_label
})
return response
self.logger.debug(
"Request Retry: Response: Valid: Retry Candidate",
extra={
'request_url': request_url,
'request_label': request_label
})
except self.request_retry_excps as retry_ex:
error_exception = retry_ex
self.logger.warning(
"Request Retry: Expected: {}: Retry Candidate".format(
base_class_name(error_exception)),
extra={
'error_details':
get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
})
if not _tries:
self.logger.error(
"Request Retry: Expected: {}: Exhausted Retries".
format(base_class_name(error_exception)))
raise
except TuneRequestBaseError as tmv_ex:
error_exception = tmv_ex
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({
'error_exception':
base_class_name(error_exception),
'error_details':
get_exception_message(error_exception),
'request_label':
request_label
})
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(tmv_ex, request_label):
self.logger.error(
"Request Retry: Integration: {}: Not Retry Candidate".
format(base_class_name(error_exception)),
extra=tmv_ex_extra)
raise
self.logger.warning(
"Request Retry: Integration: {}: Retry Candidate".format(
base_class_name(error_exception)),
extra=tmv_ex_extra)
if not _tries:
self.logger.error(
"Request Retry: Integration: {}: Exhausted Retries".
format(base_class_name(error_exception)))
raise
except Exception as ex:
error_exception = ex
ex_extra = {
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
}
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(error_exception, request_label):
self.logger.error(
"Request Retry: Unexpected: {}: Not Retry Candidate".
format(base_class_name(error_exception)),
extra=ex_extra)
raise
self.logger.warning(
"Request Retry: Unexpected: {}: Retry Candidate".format(
base_class_name(error_exception)),
extra=ex_extra)
if not _tries:
raise TuneRequestModuleError(
error_message="Unexpected: {}".format(
base_class_name(error_exception)),
errors=error_exception,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_RETRY_EXHAUSTED)
if not _tries:
self.logger.error("Request Retry: Exhausted Retries",
extra={
'attempts': _attempts,
'tries': _tries,
'request_url': request_url,
'request_label': request_label
})
raise TuneRequestModuleError(
error_message=(
"Request Retry: Exhausted Retries: {}: {}").format(
request_label, request_url),
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED)
self.logger.info("Request Retry: Performing Retry",
extra={
'tries': _tries,
'delay': _delay,
'timeout': _timeout,
'request_url': request_url,
'request_label': request_label
})
time.sleep(_delay)
if self.retry_backoff and self.retry_backoff > 0:
_delay *= self.retry_backoff
if self.retry_jitter and self.retry_jitter > 0:
_delay += self.retry_jitter
if self.retry_max_delay is not None:
_delay = min(_delay, self.retry_max_delay)
def request_json_download(
self,
request_method,
request_url,
tmp_json_file_name,
tmp_directory,
request_params=None,
request_data=None,
request_retry=None,
request_retry_func=None,
request_retry_excps=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_label=None,
build_request_curl=False,
allow_redirects=True,
verify=True,
encoding_write=None,
encoding_read=None,
):
"""Download and Read JSON file.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
tmp_json_file_name: Provide temporary name for downloaded CSV
tmp_directory: Provide temporary directory to hold downloaded CSV
request_params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_retry: (optional) Retry configuration.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable
Basic/Digest/Custom HTTP Auth.
build_request_curl: (optional) Build a copy-n-paste curl for command line
that provides same request as this call.
allow_redirects: (optional) Boolean. Set to True if
POST/PUT/DELETE redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
encoding_write:
encoding_read:
decode_unicode:
Returns:
Generator containing JSON data by rows in JSON dictionary format.
"""
_request_label = 'Request Download JSON File'
request_label = f'{request_label}: {_request_label}' if request_label is not None else _request_label
log.info(f'{request_label}: Start',
extra={
'request_url': request_url,
'encoding_write': encoding_write,
'encoding_read': encoding_read,
})
timer_start = dt.datetime.now()
_attempts = 0
_tries = 60
_delay = 10
while _tries:
_attempts += 1
log.debug(f'{request_label}: Download',
extra={
'attempts': _attempts,
'request_url': request_url,
})
response = self.mv_request.request(
request_method=request_method,
request_url=request_url,
request_params=request_params,
request_data=request_data,
request_retry=request_retry,
request_retry_func=request_retry_func,
request_retry_excps=request_retry_excps,
request_retry_excps_func=request_retry_excps_func,
request_headers=request_headers,
request_auth=request_auth,
build_request_curl=build_request_curl,
allow_redirects=allow_redirects,
verify=verify,
stream=True,
request_label=request_label)
if response is None:
log.error(f'{request_label}: No response',
extra={
'request_url': request_url,
})
raise TuneRequestModuleError(
error_message=f'{request_label}: No response',
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST)
http_status_code = response.status_code
timer_end = dt.datetime.now()
timer_delta = timer_end - timer_start
response_time_secs = timer_delta.seconds
response_headers = None
if hasattr(response, 'headers'):
response_headers = \
json.loads(
json.dumps(
dict(response.headers)
)
)
log.debug(f'{request_label}: Response Status',
extra={
'http_status_code': http_status_code,
'response_time_secs': response_time_secs,
'response_url': response.url,
'response_headers': safe_dict(response_headers),
})
if not os.path.exists(tmp_directory):
os.mkdir(tmp_directory)
tmp_json_file_path = f'{tmp_directory}/{tmp_json_file_name}'
if os.path.exists(tmp_json_file_path):
log.debug(
f'{request_label}: Removing',
extra={'file_path': tmp_json_file_path},
)
os.remove(tmp_json_file_path)
mode_write = 'wb' if encoding_write is None else 'w'
log.debug(f'{request_label}: Finished',
extra={
'file_path': tmp_json_file_path,
'mode_write': mode_write,
'encoding_write': encoding_write,
})
log.debug(f'{request_label}: Usage',
extra=env_usage(tmp_directory))
chunk_total_sum = 0
with open(file=tmp_json_file_path,
mode=mode_write,
encoding=encoding_write) as json_raw_file_w:
log.debug(f'{request_label}: Response Raw: Started',
extra={
'file_path': tmp_json_file_path,
})
_tries -= 1
error_exception = None
error_details = None
chunk_size = 8192
try:
raw_response = response.raw
while True:
chunk = raw_response.read(chunk_size,
decode_content=True)
if not chunk:
break
chunk_total_sum += chunk_size
json_raw_file_w.write(chunk)
json_raw_file_w.flush()
os.fsync(json_raw_file_w.fileno())
log.debug(f'{request_label}: By Chunk: Completed',
extra={
'file_path': tmp_json_file_path,
})
break
except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex:
error_exception = base_class_name(chunked_encoding_ex)
error_details = get_exception_message(chunked_encoding_ex)
log.warning(f'{request_label}: Error: {error_exception}',
extra={
'error_details': error_details,
'chunk_total_sum': chunk_total_sum,
})
if not _tries:
log.error(
f'{request_label}: Exhausted Retries: Error: {error_exception}'
)
raise
except http_client.IncompleteRead as incomplete_read_ex:
error_exception = base_class_name(incomplete_read_ex)
error_details = get_exception_message(incomplete_read_ex)
log.warning(f'{request_label}: IncompleteRead',
extra={
'error_exception': error_exception,
'error_details': error_details,
'chunk_total_sum': chunk_total_sum,
})
if not _tries:
log.error(
f'{request_label}: Exhausted Retries: Error: {error_exception}'
)
raise
except requests.exceptions.RequestException as request_ex:
log.error(f'{request_label}: Request Exception',
extra={
'error_exception':
base_class_name(request_ex),
'error_details':
get_exception_message(request_ex),
'chunk_total_sum':
chunk_total_sum,
})
raise
except Exception as ex:
log.error(f'{request_label}: Unexpected Exception',
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex),
'chunk_total_sum': chunk_total_sum,
})
raise
if not _tries:
log.error(f'{request_label}: Exhausted Retries',
extra={
'tries': _tries,
'request_url': request_url,
})
raise TuneRequestModuleError(
error_message=
f'{request_label}: Exhausted Retries: {request_url}',
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_RETRY_EXHAUSTED)
log.info(f'{request_label}: Performing Retry',
extra={
'tries': _tries,
'delay': _delay,
'request_url': request_url,
})
time.sleep(_delay)
tmp_json_file_size = os.path.getsize(tmp_json_file_path)
bom_enc, bom_len, bom_header = detect_bom(tmp_json_file_path)
log.info(f'{request_label}: By Chunk: Completed: Details',
extra={
'file_path': tmp_json_file_path,
'file_size': bytes_to_human(tmp_json_file_size),
'chunk_total_sum': chunk_total_sum,
'bom_encoding': bom_enc,
})
if bom_enc == 'gzip':
tmp_json_gz_file_path = f"{tmp_json_file_path}.gz"
os.rename(src=tmp_json_file_path, dst=tmp_json_gz_file_path)
with open(file=tmp_json_file_path,
mode=mode_write,
encoding=encoding_write) as json_file_w:
log.debug(f'{request_label}: GZip: Started',
extra={
'file_path': tmp_json_file_path,
})
with gzip.open(tmp_json_gz_file_path, 'r') as gzip_file_r:
json_file_w.write(gzip_file_r.read())
response_extra = {
'file_path': tmp_json_file_path,
'file_size': bytes_to_human(tmp_json_file_size),
}
log.info(f'{request_label}: Read Downloaded', extra=response_extra)
json_download = None
with open(tmp_json_file_path, mode='r') as json_file_r:
json_file_content = json_file_r.read()
try:
json_download = json.loads(json_file_content)
except ValueError as json_decode_ex:
pprint(json_file_content)
response_extra.update({
'json_file_content':
json_file_content,
'json_file_content_len':
len(json_file_content)
})
handle_json_decode_error(response_decode_ex=json_decode_ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=self.built_request_curl)
except Exception as ex:
pprint(json_file_content)
response_extra.update({
'json_file_content':
json_file_content,
'json_file_content_len':
len(json_file_content)
})
log.error(
f'{request_label}: Failed: Exception',
extra=response_extra,
)
handle_json_decode_error(response_decode_ex=ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=self.built_request_curl)
response_extra.update({'json_file_content_len': len(json_download)})
log.info(f'{request_label}: Finished', extra=response_extra)
return json_download
def download_csv(
self,
response,
tmp_directory,
tmp_csv_file_name,
request_label=None,
encoding_write=None,
decode_unicode=False,
):
_request_label = 'Download CSV'
request_label = f'{request_label}: {_request_label}' if request_label is not None else _request_label
log.debug(f'{request_label}: Start')
if not os.path.exists(tmp_directory):
os.mkdir(tmp_directory)
tmp_csv_file_path = \
f'{tmp_directory}/{tmp_csv_file_name}'
if os.path.exists(tmp_csv_file_path):
log.debug(
f'{request_label}: Removing previous CSV',
extra={'file_path': tmp_csv_file_path},
)
os.remove(tmp_csv_file_path)
mode_write = 'wb' if encoding_write is None else 'w'
log.debug(f'{request_label}: Details',
extra={
'file_path': tmp_csv_file_path,
'mode_write': mode_write,
'encoding_write': encoding_write,
})
chunk_total_sum = 0
with open(file=tmp_csv_file_path,
mode=mode_write,
encoding=encoding_write) as csv_file_wb:
log.debug(f'{request_label}: By Chunk: Started',
extra={
'file_path': tmp_csv_file_path,
'request_label': request_label
})
error_exception = None
error_details = None
try:
for chunk in response.iter_content(
chunk_size=8192, decode_unicode=decode_unicode):
if not chunk:
break
chunk_total_sum += 8192
csv_file_wb.write(chunk)
csv_file_wb.flush()
os.fsync(csv_file_wb.fileno())
log.debug(f'{request_label}: By Chunk: Completed',
extra={
'file_path': tmp_csv_file_path,
})
except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex:
error_exception = base_class_name(chunked_encoding_ex)
error_details = get_exception_message(chunked_encoding_ex)
log.warning(f'{request_label}: ChunkedEncodingError',
extra={
'error_exception': error_exception,
'error_details': error_details,
'chunk_total_sum':
bytes_to_human(chunk_total_sum),
})
return (None, 0)
except http_client.IncompleteRead as incomplete_read_ex:
error_exception = base_class_name(incomplete_read_ex)
error_details = get_exception_message(incomplete_read_ex)
log.warning(f'{request_label}: IncompleteRead',
extra={
'error_exception': error_exception,
'error_details': error_details,
'chunk_total_sum':
bytes_to_human(chunk_total_sum),
})
return (None, 0)
except requests.exceptions.RequestException as request_ex:
log.error(f'{request_label}: Request Exception',
extra={
'error_exception': base_class_name(request_ex),
'error_details':
get_exception_message(request_ex),
'chunk_total_sum':
bytes_to_human(chunk_total_sum),
})
raise
except Exception as ex:
log.error(f'{request_label}: Unexpected Exception',
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex),
'chunk_total_sum':
bytes_to_human(chunk_total_sum),
})
raise
tmp_csv_file_size = os.path.getsize(tmp_csv_file_path)
bom_enc, bom_len, bom_header = detect_bom(tmp_csv_file_path)
log.info(f'{request_label}: By Chunk: Completed: Details',
extra={
'file_path': tmp_csv_file_path,
'file_size': bytes_to_human(tmp_csv_file_size),
'chunk_total_sum': bytes_to_human(chunk_total_sum),
'bom_encoding': bom_enc
})
log.debug("Download CSV: Usage", extra=env_usage(tmp_directory))
tmp_csv_file_name_wo_ext = \
os.path.splitext(
os.path.basename(tmp_csv_file_name)
)[0]
tmp_csv_file_path_wo_bom = \
f'{tmp_directory}/{tmp_csv_file_name_wo_ext}_wo_bom.csv'
if os.path.exists(tmp_csv_file_path_wo_bom):
os.remove(tmp_csv_file_path_wo_bom)
bom_enc, bom_len = remove_bom(tmp_csv_file_path,
tmp_csv_file_path_wo_bom)
log.debug(f'{request_label}: Encoding',
extra={
'bom_enc': bom_enc,
'bom_len': bom_len
})
if bom_len > 0:
tmp_csv_file_path = tmp_csv_file_path_wo_bom
return (tmp_csv_file_path, tmp_csv_file_size)
def handle_json_decode_error(
response_decode_ex,
response,
response_extra=None,
request_label=None,
request_curl=None,
):
"""Handle JSON Decode Error
Args:
response_json_decode_error:
response:
response_extra:
request_label:
Returns:
"""
if response_extra is None:
response_extra = {}
if request_label:
response_extra.update({'request_label': request_label})
if hasattr(response, 'text') and \
response.text and \
len(response.text) > 0:
response_details = response.text
response_details_source = 'text'
response_content_length = len(response_details)
if response_details.startswith('<html'):
response_details_source = 'html'
soup_html = bs4.BeautifulSoup(response_details, "html.parser")
# kill all script and style elements
for script in soup_html(["script", "style"]):
script.extract() # rip it out
text_html = soup_html.get_text()
lines_html = [
line for line in text_html.split('\n') if line.strip() != ''
]
lines_html = [line.strip(' ') for line in lines_html]
response_details = lines_html
elif response_details.startswith('<?xml'):
response_details_source = 'xml'
response_details = json.dumps(xmltodict.parse(response_details))
else:
pprint(response_details)
response_extra.update({
'response_details':
response_details,
'response_details_source':
response_details_source,
'response_content_length':
response_content_length,
'error_exception':
base_class_name(response_decode_ex),
'error_details':
get_exception_message(response_decode_ex)
})
log.error("Validate JSON Response: Failed: Invalid", extra=response_extra)
raise TuneRequestModuleError(
error_message="Validate JSON Response: Failed: Invalid",
errors=response_decode_ex,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
