def test_empty():
assert safe_int("") == 0
assert safe_int('') == 0
assert safe_int('', 7) == 7
assert safe_float("") == 0.0
assert safe_float('') == 0.0
assert safe_float('', default=7.7) == 7.7
assert safe_dict("") == {}
assert safe_dict('') == {}
assert safe_dict('', {'Max': 'Ohad'}) == {'Max': 'Ohad'}
def test_None():
assert safe_int(None) == 0
assert safe_int(None, 7) == 7
assert safe_float(None) == 0.0
assert safe_float(None, default=7.7) == 7.7
assert safe_dict(None) == {}
assert safe_dict(None, {'Jeff': 'Tanner'}) == {'Jeff': 'Tanner'}
assert safe_str(None) == ''
assert safe_str(None, "stas") == "stas"
assert safe_cast(None, str) is None
assert safe_cast(None, str, default="TuliTuliTuli") == "TuliTuliTuli"
def test_safe_dict():
# test basic dict:
assert isinstance(safe_dict({'key': 'value'}), dict)
# test fail:
with pytest.raises(
TypeError,
message='Expecting TypeError because passing not iterable value.'):
assert safe_dict(5)
with pytest.raises(
ValueError,
message='Expecting ValueError because str not castable to dict.'):
assert safe_dict('Hello Jeff')
def test_safe_dict():
# test basic dict:
assert isinstance(safe_dict({'key': 'value'}), dict)
# test fail:
with pytest.raises(TypeError, message='Expecting TypeError because passing not iterable value.') as excinfo:
assert safe_dict(5)
assert 'Error: \'int\' object is not iterable' in str(excinfo.value)
assert 'Value: 5' in str(excinfo.value)
assert 'Cast: int to dict' in str(excinfo.value)
with pytest.raises(ValueError, message='Expecting ValueError because str not castable to dict.') as excinfo:
assert safe_dict('Hello Jeff')
assert 'Error: Dictionary update sequence element #0 has length 1; 2 is required' \
in str(excinfo.value)
assert 'Value: Hello Jeff' in str(excinfo.value)
assert 'Cast: str to dict' in str(excinfo.value)
def request_csv_download(
self,
request_method,
request_url,
tmp_csv_file_name,
tmp_directory,
request_params=None,
request_data=None,
request_retry=None,
request_retry_func=None,
request_retry_excps=None,
request_retry_http_status_codes=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_label=None,
build_request_curl=True,
allow_redirects=True,
verify=True,
skip_first_row=False,
skip_last_row=False,
read_first_row=False,
csv_delimiter=',',
csv_header=None,
encoding_write=None,
encoding_read=None,
decode_unicode=False,
):
"""Download and Read CSV file.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
tmp_csv_file_name: Provide temporary name for downloaded CSV
tmp_directory: Provide temporary directory to hold downloaded CSV
request_params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_retry: (optional) Retry configuration.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable
Basic/Digest/Custom HTTP Auth.
allow_redirects: (optional) Boolean. Set to True if
POST/PUT/DELETE redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
skip_first_row: (optional) Skip first row if it does not contain
column headers.
skip_last_row: (optional) Skip first row if it does not contain
column values.
read_first_row: (optional) Read first row separate from data returned.
csv_delimiter: (optional) Delimiter character, default comma ','.
csv_header:
encoding_write:
encoding_read:
decode_unicode:
Returns:
Generator containing CSV data by rows in JSON dictionary format.
"""
_request_label = 'Request Download CSV File'
request_label = f'{request_label}: {_request_label}' if request_label is not None else _request_label
log.debug(f'{request_label}: Start',
extra={
'request_url': request_url,
'encoding_write': encoding_write,
'encoding_read': encoding_read,
})
timer_start = dt.datetime.now()
_attempts = 0
_tries = 60
_delay = 10
while _tries:
_attempts += 1
log.info(f'{request_label}: Attempt: {_attempts}',
extra={
'request_url': request_url,
})
response = self.mv_request.request(
request_method=request_method,
request_url=request_url,
request_params=request_params,
request_data=request_data,
request_retry=request_retry,
request_retry_func=request_retry_func,
request_retry_excps=request_retry_excps,
request_retry_http_status_codes=request_retry_http_status_codes,
request_retry_excps_func=request_retry_excps_func,
request_headers=request_headers,
request_auth=request_auth,
build_request_curl=build_request_curl,
allow_redirects=allow_redirects,
verify=verify,
stream=True,
request_label=request_label)
if response is None:
log.error(f'{request_label}: No response',
extra={
'request_url': request_url,
})
raise TuneRequestModuleError(
error_message=f'{request_label}: No response',
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST,
)
http_status_code = response.status_code
timer_end = dt.datetime.now()
timer_delta = timer_end - timer_start
response_time_secs = timer_delta.seconds
response_headers = None
if hasattr(response, 'headers'):
response_headers = \
json.loads(
json.dumps(
dict(response.headers)
)
)
log.debug(f'{request_label}: Response Status',
extra={
'http_status_code': http_status_code,
'response_time_secs': response_time_secs,
'response_url': response.url,
'response_headers': safe_dict(response_headers),
})
(tmp_csv_file_path, tmp_csv_file_size) = self.download_csv(
response,
tmp_directory,
tmp_csv_file_name,
request_label=request_label,
encoding_write=encoding_write,
decode_unicode=decode_unicode)
if tmp_csv_file_path is not None:
break
_tries -= 1
if not _tries:
log.error(f'{request_label}: Exhausted Retries',
extra={
'tries': _tries,
'request_url': request_url,
})
raise TuneRequestModuleError(
error_message=f'{request_label}: Exhausted Retries',
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED)
log.info(f'{request_label}: Performing Retry',
extra={
'tries': _tries,
'delay': _delay,
'request_url': request_url,
})
time.sleep(_delay)
log.info(f'{request_label}: Finished',
extra={
'file_path': tmp_csv_file_path,
'file_size': bytes_to_human(tmp_csv_file_size),
'encoding_read': encoding_read,
})
log.debug(
f'{request_label}: Usage',
extra=env_usage(tmp_directory),
)
with open(file=tmp_csv_file_path, mode='r',
encoding=encoding_read) as csv_file_r:
if read_first_row:
csv_report_name = csv_file_r.readline()
csv_report_name = re.sub('\"', '', csv_report_name)
csv_report_name = re.sub('\n', '', csv_report_name)
log.info(
f'{request_label}: Report',
extra={'csv_report_name': csv_report_name},
)
elif skip_first_row:
next(csv_file_r)
csv_file_header = next(csv_file_r)
csv_header_actual = \
[h.strip() for h in csv_file_header.split(csv_delimiter)]
csv_header_hr = []
index = 0
for column_name in csv_header_actual:
csv_header_hr.append({'index': index, 'name': column_name})
index += 1
log.debug(
f'{request_label}: Content Header',
extra={'csv_header': csv_header_hr},
)
csv_fieldnames = csv_header if csv_header else csv_header_actual
csv_dict_reader = csv.DictReader(csv_file_r,
fieldnames=csv_fieldnames,
delimiter=csv_delimiter)
if skip_last_row:
for row in csv_skip_last_row(csv_dict_reader):
yield row
else:
for row in csv_dict_reader:
yield row
def request_json_download(
self,
request_method,
request_url,
tmp_json_file_name,
tmp_directory,
request_params=None,
request_data=None,
request_retry=None,
request_retry_func=None,
request_retry_excps=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_label=None,
build_request_curl=False,
allow_redirects=True,
verify=True,
encoding_write=None,
encoding_read=None,
):
"""Download and Read JSON file.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
tmp_json_file_name: Provide temporary name for downloaded CSV
tmp_directory: Provide temporary directory to hold downloaded CSV
request_params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_retry: (optional) Retry configuration.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable
Basic/Digest/Custom HTTP Auth.
build_request_curl: (optional) Build a copy-n-paste curl for command line
that provides same request as this call.
allow_redirects: (optional) Boolean. Set to True if
POST/PUT/DELETE redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
encoding_write:
encoding_read:
decode_unicode:
Returns:
Generator containing JSON data by rows in JSON dictionary format.
"""
_request_label = 'Request Download JSON File'
request_label = f'{request_label}: {_request_label}' if request_label is not None else _request_label
log.info(f'{request_label}: Start',
extra={
'request_url': request_url,
'encoding_write': encoding_write,
'encoding_read': encoding_read,
})
timer_start = dt.datetime.now()
_attempts = 0
_tries = 60
_delay = 10
while _tries:
_attempts += 1
log.debug(f'{request_label}: Download',
extra={
'attempts': _attempts,
'request_url': request_url,
})
response = self.mv_request.request(
request_method=request_method,
request_url=request_url,
request_params=request_params,
request_data=request_data,
request_retry=request_retry,
request_retry_func=request_retry_func,
request_retry_excps=request_retry_excps,
request_retry_excps_func=request_retry_excps_func,
request_headers=request_headers,
request_auth=request_auth,
build_request_curl=build_request_curl,
allow_redirects=allow_redirects,
verify=verify,
stream=True,
request_label=request_label)
if response is None:
log.error(f'{request_label}: No response',
extra={
'request_url': request_url,
})
raise TuneRequestModuleError(
error_message=f'{request_label}: No response',
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST)
http_status_code = response.status_code
timer_end = dt.datetime.now()
timer_delta = timer_end - timer_start
response_time_secs = timer_delta.seconds
response_headers = None
if hasattr(response, 'headers'):
response_headers = \
json.loads(
json.dumps(
dict(response.headers)
)
)
log.debug(f'{request_label}: Response Status',
extra={
'http_status_code': http_status_code,
'response_time_secs': response_time_secs,
'response_url': response.url,
'response_headers': safe_dict(response_headers),
})
if not os.path.exists(tmp_directory):
os.mkdir(tmp_directory)
tmp_json_file_path = f'{tmp_directory}/{tmp_json_file_name}'
if os.path.exists(tmp_json_file_path):
log.debug(
f'{request_label}: Removing',
extra={'file_path': tmp_json_file_path},
)
os.remove(tmp_json_file_path)
mode_write = 'wb' if encoding_write is None else 'w'
log.debug(f'{request_label}: Finished',
extra={
'file_path': tmp_json_file_path,
'mode_write': mode_write,
'encoding_write': encoding_write,
})
log.debug(f'{request_label}: Usage',
extra=env_usage(tmp_directory))
chunk_total_sum = 0
with open(file=tmp_json_file_path,
mode=mode_write,
encoding=encoding_write) as json_raw_file_w:
log.debug(f'{request_label}: Response Raw: Started',
extra={
'file_path': tmp_json_file_path,
})
_tries -= 1
error_exception = None
error_details = None
chunk_size = 8192
try:
raw_response = response.raw
while True:
chunk = raw_response.read(chunk_size,
decode_content=True)
if not chunk:
break
chunk_total_sum += chunk_size
json_raw_file_w.write(chunk)
json_raw_file_w.flush()
os.fsync(json_raw_file_w.fileno())
log.debug(f'{request_label}: By Chunk: Completed',
extra={
'file_path': tmp_json_file_path,
})
break
except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex:
error_exception = base_class_name(chunked_encoding_ex)
error_details = get_exception_message(chunked_encoding_ex)
log.warning(f'{request_label}: Error: {error_exception}',
extra={
'error_details': error_details,
'chunk_total_sum': chunk_total_sum,
})
if not _tries:
log.error(
f'{request_label}: Exhausted Retries: Error: {error_exception}'
)
raise
except http_client.IncompleteRead as incomplete_read_ex:
error_exception = base_class_name(incomplete_read_ex)
error_details = get_exception_message(incomplete_read_ex)
log.warning(f'{request_label}: IncompleteRead',
extra={
'error_exception': error_exception,
'error_details': error_details,
'chunk_total_sum': chunk_total_sum,
})
if not _tries:
log.error(
f'{request_label}: Exhausted Retries: Error: {error_exception}'
)
raise
except requests.exceptions.RequestException as request_ex:
log.error(f'{request_label}: Request Exception',
extra={
'error_exception':
base_class_name(request_ex),
'error_details':
get_exception_message(request_ex),
'chunk_total_sum':
chunk_total_sum,
})
raise
except Exception as ex:
log.error(f'{request_label}: Unexpected Exception',
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex),
'chunk_total_sum': chunk_total_sum,
})
raise
if not _tries:
log.error(f'{request_label}: Exhausted Retries',
extra={
'tries': _tries,
'request_url': request_url,
})
raise TuneRequestModuleError(
error_message=
f'{request_label}: Exhausted Retries: {request_url}',
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_RETRY_EXHAUSTED)
log.info(f'{request_label}: Performing Retry',
extra={
'tries': _tries,
'delay': _delay,
'request_url': request_url,
})
time.sleep(_delay)
tmp_json_file_size = os.path.getsize(tmp_json_file_path)
bom_enc, bom_len, bom_header = detect_bom(tmp_json_file_path)
log.info(f'{request_label}: By Chunk: Completed: Details',
extra={
'file_path': tmp_json_file_path,
'file_size': bytes_to_human(tmp_json_file_size),
'chunk_total_sum': chunk_total_sum,
'bom_encoding': bom_enc,
})
if bom_enc == 'gzip':
tmp_json_gz_file_path = f"{tmp_json_file_path}.gz"
os.rename(src=tmp_json_file_path, dst=tmp_json_gz_file_path)
with open(file=tmp_json_file_path,
mode=mode_write,
encoding=encoding_write) as json_file_w:
log.debug(f'{request_label}: GZip: Started',
extra={
'file_path': tmp_json_file_path,
})
with gzip.open(tmp_json_gz_file_path, 'r') as gzip_file_r:
json_file_w.write(gzip_file_r.read())
response_extra = {
'file_path': tmp_json_file_path,
'file_size': bytes_to_human(tmp_json_file_size),
}
log.info(f'{request_label}: Read Downloaded', extra=response_extra)
json_download = None
with open(tmp_json_file_path, mode='r') as json_file_r:
json_file_content = json_file_r.read()
try:
json_download = json.loads(json_file_content)
except ValueError as json_decode_ex:
pprint(json_file_content)
response_extra.update({
'json_file_content':
json_file_content,
'json_file_content_len':
len(json_file_content)
})
handle_json_decode_error(response_decode_ex=json_decode_ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=self.built_request_curl)
except Exception as ex:
pprint(json_file_content)
response_extra.update({
'json_file_content':
json_file_content,
'json_file_content_len':
len(json_file_content)
})
log.error(
f'{request_label}: Failed: Exception',
extra=response_extra,
)
handle_json_decode_error(response_decode_ex=ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=self.built_request_curl)
response_extra.update({'json_file_content_len': len(json_download)})
log.info(f'{request_label}: Finished', extra=response_extra)
return json_download
def _check_v3_job_status_on_queue(self, export_job, request_retry=None, request_label="TMC v3 Job Status On Queue"):
"""Status of Export Report.
Args:
export_job:
request_retry:
Returns:
"""
request_label = "v3 Logs Advertisers Check Export Status"
request_url = \
self.tune_mat_request_path(
mat_api_version="v3",
controller=self.controller,
action="exports/{}".format(
export_job
)
)
self.logger.info((
"TMC v3 Logs Advertisers Base: Check Export Status: "
"Logs '{}': "
"Action: 'exports status', "
"Status of Export Report for "
"Job Handle: '{}'"
).format(self.logs_advertisers_type, export_job))
tries = -1 # default: -1 (indefinite)
delay = 10
jitter = 0
max_delay = 60
if request_retry:
if 'delay' in request_retry:
delay = request_retry['delay']
if 'jitter' in request_retry:
jitter = request_retry['jitter']
if 'max_delay' in request_retry:
max_delay = request_retry['max_delay']
request_params = {"session_token": self.session_token}
self.logger.debug("TMC v3 Logs Advertisers Base: Check Export Status", extra={'request_url': request_url})
self.logger.debug(
"TMC v3 Logs Advertisers Base: Check Export Status: Request Retry",
extra={'tries': tries,
'delay': delay,
'jitter': jitter,
'max_delay': max_delay}
)
self.logger.debug(
"TMC v3 Logs Advertisers Base: Check Export Status: Request",
extra={'request_params': safe_dict(request_params)}
)
report_url = None
_attempts = 1
export_percent_complete = 0
export_status_action = 'exports status'
self.logger.warning(
"TMC v3 Logs Advertisers Base: Check Export Status",
extra={'job': export_job,
'attempt': _attempts,
'action': export_status_action}
)
time.sleep(10)
_tries, _delay = tries, delay
while True:
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v3_request_retry_func,
request_retry_excps_func=None,
request_label=request_label
)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra={'error': get_exception_message(ex)}
)
raise
http_status_successful = is_http_status_type(
http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
if not http_status_successful:
raise TuneReportingError(
error_message="Failed to get export status on queue: {}".format(response.status_code),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST
)
json_response = response.json()
export_percent_complete = 0
if "percent_complete" in json_response:
export_percent_complete = \
safe_int(json_response["percent_complete"])
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status",
extra={
'job': export_job,
'response_status_code': json_response["status"],
'export_percent_complete': safe_int(export_percent_complete)
}
)
if (export_percent_complete == 100 and json_response["status"] == "complete" and json_response["url"]):
report_url = json_response["url"]
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status: Completed",
extra={
'job': export_job,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': safe_int(export_percent_complete)
}
)
break
if tries >= 0:
_tries -= 1
if _tries == 0:
self.logger.error(("TMC v3 Logs Advertisers Base: "
"Check Job Export Status: Exhausted Retries"),
extra={
'attempt': _attempts,
'tries': _tries,
'action': export_status_action,
'request_label': request_label,
'export_percent_complete': export_percent_complete
})
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Check Job Export Status: "
"Exhausted Retries: "
"Percent Completed: {}"
).format(export_percent_complete),
error_code=TuneReportingErrorCodes.REP_ERR_JOB_STOPPED
)
_attempts += 1
self.logger.warning(
"TMC v3 Logs Advertisers Base: Check Export Status",
extra={'attempt': _attempts,
'job': export_job,
'delay': _delay,
'action': 'exports status'}
)
time.sleep(_delay)
_delay += jitter
_delay = min(_delay, max_delay)
if export_percent_complete == 100 and not report_url:
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: Check Job Export Status: "
"Download report URL: Undefined"
)
)
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status: Finished",
extra={
'attempt': _attempts,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': export_percent_complete,
'job': export_job
}
)
return report_url
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise TuneRequestValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise TuneRequestValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
'{}: Session: Details'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug('{}: Details'.format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.debug(
'{}: Curl'.format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url})
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.tune_request.request(**kwargs)
except Exception as ex:
self.logger.error(
'{}: Request Base: Error'.format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
'{}: Response: Failed'.format(request_label),
extra={'request_curl': self.built_request_curl},
)
raise TuneRequestModuleError(
error_message='{}: Response: Failed'.format(request_label),
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
http_status_code_to_type(http_status_code)
http_status_desc = \
http_status_code_to_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
'{}: Response: Details'.format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
'{}: Cookie Payload'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled)
raise TuneRequestModuleError(**kwargs)
def _check_v2_job_status_on_queue(
self,
auth_type,
auth_value,
export_status_controller,
export_status_action,
export_job_id,
request_retry=None,
):
"""Check Job Export Status
Args:
export_status_controller:
export_status_action:
export_job_id:
request_retry:
Returns:
"""
request_label = "TMC v2 Advertiser Stats: Check Export Status"
v2_export_status_request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=export_status_controller,
action=export_status_action
)
request_params = {auth_type: auth_value, "job_id": export_job_id}
self.logger.info(
("TMC v2 Advertiser Stats: Check Job Status"),
extra={
'action': export_status_action,
'job_id': export_job_id,
'request_url': v2_export_status_request_url,
'request_params': safe_dict(request_params)
})
tries = 60 # -1 (indefinite)
delay = 10
jitter = 10
max_delay = 60
if request_retry is not None:
if 'delay' in request_retry:
delay = request_retry['delay']
if 'jitter' in request_retry:
jitter = request_retry['jitter']
if 'max_delay' in request_retry:
max_delay = request_retry['max_delay']
if 'tries' in request_retry:
tries = request_retry['tries']
else:
request_retry.update({'tries': 60})
else:
request_retry = {'tries': 60, 'delay': 10, 'timeout': 60}
self.logger.debug(msg=("TMC v2 Advertiser Stats: Check Job Status: "
"Request Retry"),
extra=request_retry)
report_url = None
_attempts = 1
export_percent_complete = 0
time.sleep(10)
_tries, _delay = tries, delay
while True:
try:
response = self.mv_request.request(
request_method="GET",
request_url=v2_export_status_request_url,
request_params=request_params,
request_label=request_label,
request_retry_func=self.tune_v2_request_retry_func)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: {}".format(
get_exception_message(ex)))
raise
http_status_successful = is_http_status_type(
http_status_code=response.status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
if not http_status_successful:
raise TuneReportingError(
error_message=(
"Failed to get export status on queue: {}").format(
response.status_code),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
if hasattr(response, 'url'):
self.logger.info(
"TMC v2 Advertiser Stats: Reporting API: Status URL",
extra={'response_url': response.url})
json_response = response.json()
if not json_response:
request_status_successful = False
elif 'status_code' not in json_response:
request_status_successful = False
else:
status_code = json_response['status_code']
request_status_successful = is_http_status_type(
http_status_code=status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
errors = None
if 'errors' in json_response:
errors = json_response['errors']
if not request_status_successful:
error_message = (
"TMC v2 Advertiser Stats: Check Job Status: GET '{}', Failed: {}, {}"
).format(v2_export_status_request_url, status_code, errors)
if (status_code ==
TuneReportingError.EX_SRV_ERR_500_INTERNAL_SERVER):
self.logger.error(error_message)
elif (status_code ==
TuneReportingError.EX_SRV_ERR_503_SERVICE_UNAVAILABLE):
self.logger.error(error_message)
elif (status_code ==
TuneReportingError.EX_SRV_ERR_504_SERVICE_TIMEOUT):
self.logger.error(error_message)
continue
elif (status_code ==
TuneReportingError.EX_CLT_ERR_408_REQUEST_TIMEOUT):
self.logger.error(
"GET '{}' request timeout, Retrying: {}".format(
v2_export_status_request_url, status_code))
continue
else:
raise TuneReportingError(error_message=error_message,
error_code=status_code)
if tries >= 0 and _tries <= 1:
if (status_code == HttpStatusCode.GATEWAY_TIMEOUT):
raise TuneReportingError(
error_message=error_message,
error_code=TuneReportingErrorCodes.GATEWAY_TIMEOUT)
elif (status_code == HttpStatusCode.REQUEST_TIMEOUT):
raise TuneReportingError(
error_message=error_message,
error_code=TuneReportingErrorCodes.REQUEST_TIMEOUT)
else:
raise TuneReportingError(error_message=error_message,
error_code=status_code)
else:
self.logger.warning(error_message)
export_percent_complete = 0
if 'data' in json_response and json_response['data']:
json_data = json_response['data']
if "percent_complete" in json_data:
export_percent_complete = \
safe_int(json_data["percent_complete"])
self.logger.info(msg=("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"Response Success"),
extra={
'job_id':
export_job_id,
'export_status':
json_data["status"],
'export_percent_complete':
safe_int(export_percent_complete),
'attempt':
_attempts
})
if (export_status_action ==
TuneV2AdvertiserStatsStatusAction.STATUS):
if (export_percent_complete == 100
and json_data["status"] == "complete"
and json_data["url"]):
report_url = json_data["url"]
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Completed"),
extra={
'job_id': export_job_id,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label
})
break
elif (export_status_action ==
TuneV2AdvertiserStatsStatusAction.DOWNLOAD):
if (export_percent_complete == 100
and json_data["status"] == "complete"
and json_data["data"]["url"]):
report_url = json_data["data"]["url"]
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Completed"),
extra={
'job_id': export_job_id,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label
})
break
else:
self.logger.debug("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"No Data Available")
if tries >= 0:
_tries -= 1
if _tries == 0:
self.logger.error(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Exhausted Retries"),
extra={
'attempt':
_attempts,
'tries':
_tries,
'action':
export_status_action,
'request_label':
request_label,
'export_percent_complete':
safe_int(export_percent_complete),
'job_id':
export_job_id
})
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"Exhausted Retries: "
"Percent Completed: {}").format(
safe_int(export_percent_complete)),
error_code=TuneReportingErrorCodes.
REP_ERR_RETRY_EXHAUSTED)
_attempts += 1
self.logger.info("TMC v2 Advertiser Stats: Check Job Status",
extra={
'attempt': _attempts,
'job_id': export_job_id,
'delay': _delay,
'action': export_status_action
})
time.sleep(_delay)
_delay += jitter
_delay = min(_delay, max_delay)
if export_percent_complete == 100 and not report_url:
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: Check Job Export Status: "
"Download report URL: Undefined"),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
self.logger.info(
"TMC v2 Advertiser Stats: Check Job Export Status: Finished",
extra={
'attempt': _attempts,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': export_percent_complete,
'job_id': export_job_id
})
return report_url
def _process_export_stream_v2(
self,
auth_type_use,
str_date_start,
str_date_end,
export_controller,
export_action,
export_status_controller,
export_status_action,
request_params,
request_retry,
request_label="TMC v2 Advertiser Stats Export Stream",
):
"""Process Export Job by Steaming
Args:
str_date_start:
str_date_end:
export_controller:
export_action:
export_status_controller:
export_status_action:
request_params:
request_retry:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
str_date_start += " 00:00:00"
str_date_end += " 23:59:59"
request_params["start_date"] = \
str_date_start
request_params["end_date"] = \
str_date_end
self.logger.debug(
"TMC v2 Advertiser Stats: Export Stream V2: Export Job to Queue",
extra={
'controller': export_controller,
'action': export_action,
'request_params': safe_dict(request_params),
'request_retry': safe_dict(request_retry)
})
export_job_id = self._export_v2_job_to_queue(
export_controller,
export_action,
request_params,
request_retry,
)
self.logger.debug(
"TMC v2 Advertiser Stats: Export Stream V2: Check Job status on Queue",
extra={
'controller': export_status_controller,
'action': export_status_action,
'job_id': export_job_id
})
export_report_url = self._check_v2_job_status_on_queue(
auth_type_use,
auth_value,
export_status_controller,
export_status_action,
export_job_id,
request_retry=request_retry)
if not export_report_url:
raise TuneReportingError(
error_message="Export URL not defined",
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE,
)
self.logger.info(("TMC v2 Advertiser Stats: "
"Export Stream V2: Request Completed Job"),
extra={
'job_id': export_job_id,
'report_url': export_report_url
})
response = self.mv_request.request(request_method="GET",
request_url=export_report_url,
stream=True,
request_label=request_label)
self.logger.info(
"TMC v2 Advertiser Stats: Export Stream V2: Response Completed Job",
extra={
'response_status_code': response.status_code,
'response_headers': response.headers,
'job_id': export_job_id,
'report_url': export_report_url
})
return response
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
request_cert=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
request_cert: (optional) Cert tuple to enable Client side certificates.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise RequestsFortifiedValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise RequestsFortifiedValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
"{0}: Session: Details".format(request_label),
extra={
'cookie_payload': self.requests_session_client.session.cookies.get_dict(),
'request_label': request_label
}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug("{0}: Details".format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if request_cert:
kwargs.update({'cert': request_cert})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.note(
"{0}: Curl".format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug(
"{0}: {1}".format(request_label, request_method),
extra={'response_url': response.url}
)
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.requests_session_client.request(**kwargs)
except Exception as ex:
self.logger.error(
"{0}: Request Base: Error".format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
"{0}: Response: Failed".format(request_label),
extra={
'request_curl': self.built_request_curl
},
)
raise RequestsFortifiedModuleError(
error_message="{0}: Response: Failed".format(request_label),
error_code=RequestsFortifiedErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
"{0}: Response: Details".format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
"{0}: Cookie Payload".format(request_label),
extra={
'cookie_payload': self.requests_session_client.session.cookies.get_dict(),
'request_label': request_label
}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error("{0}: Response: Failed".format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error("{0}: Error: Response: Details".format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise RequestsFortifiedClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise RequestsFortifiedServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error("{0}: Error: Unhandled".format(request_label), extra=extra_unhandled)
raise RequestsFortifiedModuleError(**kwargs)
def request_csv_download(
self,
request_method,
request_url,
tmp_csv_file_name,
tmp_directory,
request_params=None,
request_data=None,
request_retry=None,
request_retry_func=None,
request_retry_excps=None,
request_retry_http_status_codes=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_label=None,
build_request_curl=True,
allow_redirects=True,
verify=True,
skip_first_row=False,
skip_last_row=False,
read_first_row=False,
csv_delimiter=',',
csv_header=None,
encoding_write=None,
encoding_read=None,
decode_unicode=False,
):
"""Download and Read CSV file.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
tmp_csv_file_name: Provide temporary name for downloaded CSV
tmp_directory: Provide temporary directory to hold downloaded CSV
request_params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_retry: (optional) Retry configuration.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable
Basic/Digest/Custom HTTP Auth.
allow_redirects: (optional) Boolean. Set to True if
POST/PUT/DELETE redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
skip_first_row: (optional) Skip first row if it does not contain
column headers.
skip_last_row: (optional) Skip first row if it does not contain
column values.
read_first_row: (optional) Read first row separate from data returned.
csv_delimiter: (optional) Delimiter character, default comma ','.
csv_header:
encoding_write:
encoding_read:
decode_unicode:
Returns:
Generator containing CSV data by rows in JSON dictionary format.
"""
_request_label = 'Request Download CSV File'
request_label = "{0}: {1}".format(request_label, _request_label) if request_label is not None else _request_label
log.debug(
"{0}: Start".format(request_label),
extra={
'request_url': request_url,
'encoding_write': encoding_write,
'encoding_read': encoding_read,
}
)
timer_start = dt.datetime.now()
_attempts = 0
_tries = 60
_delay = 10
while _tries:
_attempts += 1
log.info(
"{0}: Attempt: {1}".format(request_label, _attempts),
extra={
'request_url': request_url,
}
)
response = self.requests_client.request(
request_method=request_method,
request_url=request_url,
request_params=request_params,
request_data=request_data,
request_retry=request_retry,
request_retry_func=request_retry_func,
request_retry_excps=request_retry_excps,
request_retry_http_status_codes=request_retry_http_status_codes,
request_retry_excps_func=request_retry_excps_func,
request_headers=request_headers,
request_auth=request_auth,
build_request_curl=build_request_curl,
allow_redirects=allow_redirects,
verify=verify,
stream=True,
request_label=request_label
)
if response is None:
log.error(
"{0}: No response".format(request_label),
extra={
'request_url': request_url,
}
)
raise RequestsFortifiedModuleError(
error_message="{0}: No response".format(request_label),
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST,
)
http_status_code = response.status_code
timer_end = dt.datetime.now()
timer_delta = timer_end - timer_start
response_time_secs = timer_delta.seconds
response_headers = None
if hasattr(response, 'headers'):
response_headers = \
json.loads(
json.dumps(
dict(response.headers)
)
)
log.debug(
"{0}: Response Status".format(request_label),
extra={
'http_status_code': http_status_code,
'response_time_secs': response_time_secs,
'response_url': response.url,
'response_headers': safe_dict(response_headers),
}
)
(tmp_csv_file_path, tmp_csv_file_size) = self.download_csv(
response,
tmp_directory,
tmp_csv_file_name,
request_label=request_label,
encoding_write=encoding_write,
decode_unicode=decode_unicode
)
if tmp_csv_file_path is not None:
break
_tries -= 1
if not _tries:
log.error(
"{0}: Exhausted Retries".format(request_label),
extra={
'tries': _tries,
'request_url': request_url,
}
)
raise RequestsFortifiedModuleError(
error_message="{0}: Exhausted Retries".format(request_label),
error_code=RequestsFortifiedErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
log.info(
"{0}: Performing Retry".format(request_label),
extra={
'tries': _tries,
'delay': _delay,
'request_url': request_url,
}
)
time.sleep(_delay)
log.info(
"{0}: Finished".format(request_label),
extra={
'file_path': tmp_csv_file_path,
'file_size': bytes_to_human(tmp_csv_file_size),
'encoding_read': encoding_read,
}
)
log.debug(
"{0}: Usage".format(request_label),
extra=env_usage(tmp_directory),
)
with open(file=tmp_csv_file_path, mode='r', encoding=encoding_read) as csv_file_r:
if read_first_row:
csv_report_name = csv_file_r.readline()
csv_report_name = re.sub('\"', '', csv_report_name)
csv_report_name = re.sub('\n', '', csv_report_name)
log.info(
"{0}: Report".format(request_label),
extra={'csv_report_name': csv_report_name},
)
elif skip_first_row:
next(csv_file_r)
csv_file_header = next(csv_file_r)
csv_header_actual = \
[h.strip() for h in csv_file_header.split(csv_delimiter)]
csv_header_hr = []
index = 0
for column_name in csv_header_actual:
csv_header_hr.append({'index': index, 'name': column_name})
index += 1
log.debug(
"{0}: Content Header".format(request_label),
extra={'csv_header': csv_header_hr},
)
csv_fieldnames = csv_header if csv_header else csv_header_actual
csv_dict_reader = csv.DictReader(csv_file_r, fieldnames=csv_fieldnames, delimiter=csv_delimiter)
if skip_last_row:
for row in csv_skip_last_row(csv_dict_reader):
yield row
else:
for row in csv_dict_reader:
yield row
def request_json_download(
self,
request_method,
request_url,
tmp_json_file_name,
tmp_directory,
request_params=None,
request_data=None,
request_retry=None,
request_retry_func=None,
request_retry_excps=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
request_label=None,
build_request_curl=False,
allow_redirects=True,
verify=True,
encoding_write=None,
encoding_read=None,
):
"""Download and Read JSON file.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
tmp_json_file_name: Provide temporary name for downloaded CSV
tmp_directory: Provide temporary directory to hold downloaded CSV
request_params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_retry: (optional) Retry configuration.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable
Basic/Digest/Custom HTTP Auth.
build_request_curl: (optional) Build a copy-n-paste curl for command line
that provides same request as this call.
allow_redirects: (optional) Boolean. Set to True if
POST/PUT/DELETE redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
encoding_write:
encoding_read:
decode_unicode:
Returns:
Generator containing JSON data by rows in JSON dictionary format.
"""
_request_label = "Request Download JSON File"
request_label = "{0}: {1}".format(request_label, _request_label) if request_label is not None else _request_label
log.info(
"{0}: Start".format(request_label),
extra={
'request_url': request_url,
'encoding_write': encoding_write,
'encoding_read': encoding_read,
}
)
timer_start = dt.datetime.now()
_attempts = 0
_tries = 60
_delay = 10
while _tries:
_attempts += 1
log.debug(
"{0}: Download".format(request_label),
extra={
'attempts': _attempts,
'request_url': request_url,
}
)
response = self.requests_client.request(
request_method=request_method,
request_url=request_url,
request_params=request_params,
request_data=request_data,
request_retry=request_retry,
request_retry_func=request_retry_func,
request_retry_excps=request_retry_excps,
request_retry_excps_func=request_retry_excps_func,
request_headers=request_headers,
request_auth=request_auth,
build_request_curl=build_request_curl,
allow_redirects=allow_redirects,
verify=verify,
stream=True,
request_label=request_label
)
if response is None:
log.error(
"{0}: No response".format(request_label),
extra={
'request_url': request_url,
}
)
raise RequestsFortifiedModuleError(
error_message="{0}: No response".format(request_label),
error_code=RequestsFortifiedErrorCodes.REQ_ERR_REQUEST
)
http_status_code = response.status_code
timer_end = dt.datetime.now()
timer_delta = timer_end - timer_start
response_time_secs = timer_delta.seconds
response_headers = None
if hasattr(response, 'headers'):
response_headers = \
json.loads(
json.dumps(
dict(response.headers)
)
)
log.debug(
"{0}: Response Status".format(request_label),
extra={
'http_status_code': http_status_code,
'response_time_secs': response_time_secs,
'response_url': response.url,
'response_headers': safe_dict(response_headers),
}
)
if not os.path.exists(tmp_directory):
os.mkdir(tmp_directory)
tmp_json_file_path = "{0}/{1}".format(tmp_directory, tmp_json_file_name)
if os.path.exists(tmp_json_file_path):
log.debug(
"{0}: Removing".format(request_label),
extra={'file_path': tmp_json_file_path},
)
os.remove(tmp_json_file_path)
mode_write = 'wb' if encoding_write is None else 'w'
log.debug(
"{0}: Finished".format(request_label),
extra={
'file_path': tmp_json_file_path,
'mode_write': mode_write,
'encoding_write': encoding_write,
}
)
log.debug(
"{0}: Usage".format(request_label),
extra=env_usage(tmp_directory)
)
chunk_total_sum = 0
with open(file=tmp_json_file_path, mode=mode_write, encoding=encoding_write) as json_raw_file_w:
log.debug(
"{0}: Response Raw: Started".format(request_label),
extra={
'file_path': tmp_json_file_path,
}
)
_tries -= 1
error_exception = None
error_details = None
chunk_size = 8192
try:
raw_response = response.raw
while True:
chunk = raw_response.read(chunk_size, decode_content=True)
if not chunk:
break
chunk_total_sum += chunk_size
json_raw_file_w.write(chunk)
json_raw_file_w.flush()
os.fsync(json_raw_file_w.fileno())
log.debug(
"{0}: By Chunk: Completed".format(request_label),
extra={
'file_path': tmp_json_file_path,
}
)
break
except requests.exceptions.ChunkedEncodingError as chunked_encoding_ex:
error_exception = base_class_name(chunked_encoding_ex)
error_details = get_exception_message(chunked_encoding_ex)
log.warning(
"{0}: Error: {1}".format(request_label, error_exception),
extra={
'error_details': error_details,
'chunk_total_sum': chunk_total_sum,
}
)
if not _tries:
log.error(
"{0}: Exhausted Retries: Error: {1}".format(request_label, error_exception),
)
raise
except http_client.IncompleteRead as incomplete_read_ex:
error_exception = base_class_name(incomplete_read_ex)
error_details = get_exception_message(incomplete_read_ex)
log.warning(
"{0}: IncompleteRead".format(request_label),
extra={
'error_exception': error_exception,
'error_details': error_details,
'chunk_total_sum': chunk_total_sum,
}
)
if not _tries:
log.error(
"{0}: Exhausted Retries: Error: {1}".format(request_label, error_exception),
)
raise
except requests.exceptions.RequestException as request_ex:
log.error(
"{0}: Request Exception".format(request_label),
extra={
'error_exception': base_class_name(request_ex),
'error_details': get_exception_message(request_ex),
'chunk_total_sum': chunk_total_sum,
}
)
raise
except Exception as ex:
log.error(
"{0}: Unexpected Exception".format(request_label),
extra={
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex),
'chunk_total_sum': chunk_total_sum,
}
)
raise
if not _tries:
log.error(
"{0}: Exhausted Retries".format(request_label),
extra={
'tries': _tries,
'request_url': request_url,
}
)
raise RequestsFortifiedModuleError(
error_message="{0}: Exhausted Retries: {1}".format(request_label, request_url),
error_request_curl=self.built_request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
log.info(
"{0}: Performing Retry".format(request_label),
extra={
'tries': _tries,
'delay': _delay,
'request_url': request_url,
}
)
time.sleep(_delay)
tmp_json_file_size = os.path.getsize(tmp_json_file_path)
bom_enc, bom_len, bom_header = detect_bom(tmp_json_file_path)
log.info(
"{0}: By Chunk: Completed: Details".format(request_label),
extra={
'file_path': tmp_json_file_path,
'file_size': bytes_to_human(tmp_json_file_size),
'chunk_total_sum': chunk_total_sum,
'bom_encoding': bom_enc,
}
)
if bom_enc == 'gzip':
tmp_json_gz_file_path = "%s.gz" % tmp_json_file_path
os.rename(src=tmp_json_file_path, dst=tmp_json_gz_file_path)
with open(file=tmp_json_file_path, mode=mode_write, encoding=encoding_write) as json_file_w:
log.debug(
"{0}: GZip: Started".format(request_label),
extra={
'file_path': tmp_json_file_path,
}
)
with gzip.open(tmp_json_gz_file_path, 'r') as gzip_file_r:
json_file_w.write(gzip_file_r.read())
response_extra = {
'file_path': tmp_json_file_path,
'file_size': bytes_to_human(tmp_json_file_size),
}
log.info(
"{0}: Read Downloaded".format(request_label),
extra=response_extra
)
json_download = None
with open(tmp_json_file_path, mode='r') as json_file_r:
json_file_content = json_file_r.read()
try:
json_download = json.loads(json_file_content)
except ValueError as json_decode_ex:
pprint(json_file_content)
response_extra.update({
'json_file_content': json_file_content,
'json_file_content_len': len(json_file_content)
})
handle_json_decode_error(
response_decode_ex=json_decode_ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=self.built_request_curl
)
except Exception as ex:
pprint(json_file_content)
response_extra.update({
'json_file_content': json_file_content,
'json_file_content_len': len(json_file_content)
})
log.error(
"{0}: Failed: Exception".format(request_label),
extra=response_extra,
)
handle_json_decode_error(
response_decode_ex=ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=self.built_request_curl
)
response_extra.update({'json_file_content_len': len(json_download)})
log.info(
"{0}: Finished".format(request_label),
extra=response_extra
)
return json_download
print(safe_cost(.12))
print(safe_cost(.123))
print(safe_cost(.1234))
print(safe_cost(.12345))
print(safe_cost(4))
print(safe_cost(4.1))
print(safe_cost(4.12))
print(safe_cost(4.123))
print(safe_cost(4.1234))
print(safe_cost(4.12345))
print("\n")
print('==================================')
print('safe_dict()')
print('==================================')
print(safe_dict({'key': 'value'}))
print("\n")
print('==================================')
print('safe_fraction()')
print('==================================')
print(safe_fraction("1/2"))
print("\n")
print('==================================')
print('ValueError')
print('==================================')
try:
safe_int("apple")
except ValueError as ex:
print(str(ex))
def _map_request_params(self,
auth_type_use,
start_date,
end_date,
request_params=None):
"""Build Request Paramaters
Args:
start_date:
end_date:
request_params:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
dict_request_params = {
auth_type_use:
auth_value,
"source":
"multiverse",
"response_timezone":
self.timezone,
"timestamp":
"datehour",
"group": ("advertiser_id,"
"country_id,"
"currency_code,"
"is_reengagement,"
"platform,"
"publisher_id,"
"publisher_sub_ad_id,"
"publisher_sub_adgroup_id,"
"publisher_sub_campaign_id,"
"publisher_sub_publisher_id,"
"publisher_sub_site_id,"
"purchase_validation_status,"
"site_id"),
"fields": ("ad_clicks,"
"ad_clicks_unique,"
"ad_impressions,"
"ad_impressions_unique,"
"ad_network_id,"
"advertiser_id,"
"conversions,"
"country.code,"
"country.name,"
"currency_code,"
"date_hour,"
"events,"
"installs,"
"is_reengagement,"
"payouts,"
"publisher.name,"
"publisher_id,"
"publisher_sub_ad.ref,"
"publisher_sub_adgroup.ref,"
"publisher_sub_campaign.ref,"
"publisher_sub_publisher.ref,"
"publisher_sub_site.ref,"
"site.mobile_app_type,"
"site.package_name,"
"site.store_app_id,"
"site_id"),
"filter":
"({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA),
"start_date":
start_date,
"end_date":
end_date,
"debug":
0
}
if request_params:
self.logger.debug(
"TuneV2AdvertiserStatsActuals: Request",
extra={'request_params': safe_dict(request_params)})
if "fields" in request_params and \
request_params["fields"]:
dict_request_params["fields"] = \
request_params["fields"]
if "group" in request_params and \
request_params["group"]:
dict_request_params["group"] = \
request_params["group"]
if "timestamp" in request_params and \
request_params["timestamp"]:
dict_request_params["timestamp"] = \
request_params["timestamp"]
if "filter" in request_params and \
request_params["filter"]:
dict_request_params["filter"] = "({} AND {})".format(
request_params["filter"],
self._FILTER_NOT_DEBUG_NOR_TEST_DATA)
if "format" in request_params:
dict_request_params["format"] = \
request_params["format"]
if "offset" in request_params:
dict_request_params["offset"] = \
int(request_params["offset"])
if "page" in request_params:
dict_request_params["page"] = \
int(request_params["page"])
if "limit" in request_params:
dict_request_params["limit"] = \
int(request_params["limit"])
if "debug" in request_params:
dict_request_params["debug"] = \
int(request_params["debug"])
response_timezone = None
if "timezone" in request_params:
response_timezone = request_params["timezone"]
if "response_timezone" in request_params:
response_timezone = request_params["response_timezone"]
if response_timezone:
if not validate_tz_name(response_timezone):
return TuneReportingError(
error_message="Invalid Timezone: {}".format(
response_timezone))
self.timezone = response_timezone
dict_request_params["response_timezone"] = \
self.timezone
self.logger.debug(("TuneV2AdvertiserStatsActuals: "
"Timezone: {}").format(self.timezone))
return dict_request_params