def test_None():
assert safe_int(None) == 0
assert safe_int(None, 7) == 7
assert safe_float(None) == 0.0
assert safe_float(None, default=7.7) == 7.7
assert safe_dict(None) == {}
assert safe_dict(None, {'Jeff': 'Tanner'}) == {'Jeff': 'Tanner'}
assert safe_str(None) == ''
assert safe_str(None, "stas") == "stas"
assert safe_cast(None, str) is None
assert safe_cast(None, str, default="TuliTuliTuli") == "TuliTuliTuli"
def write_user_config(userconfig, *args):
if (safe_str(args[0], default=False)):
write_path = uni.Path(fg.config_path, args[0])
else:
write_path = uni.Path(fg.config_path, fg.today)
fluid_dump(write_path, userconfig)
print("User-config written.")
def format(self, record):
fmt_record_items = {}
for key, meta in TUNE_STANDARD_FORMAT_SUPPORTED_KEYS.items():
if key in record.__dict__:
if 'label' in meta and meta['label']:
key_lower = str(key).lower()
fmt_record_item = "{}: ".format(key_lower)
else:
fmt_record_item = ""
fmt_record_item.lower()
if 'fmt' in meta:
meta_fmt = meta['fmt']
else:
meta_fmt = '{},'
try:
if meta['type'] == str:
fmt_record_item += \
"\"{}\", ".format(safe_str(record.__dict__[key]))
elif meta['type'] == int:
fmt_record_item += \
"{:,}, ".format(safe_int(record.__dict__[key]))
elif meta['type'] == float:
fmt_record_item += \
"{0:.2f}, ".format(safe_float(record.__dict__[key]))
elif meta['type'] == dict:
fmt_record_item += \
"{}, ".format(
json.loads(
json.dumps(dict(record.__dict__[key]))
)
)
else:
fmt_record_item += meta_fmt.format(
record.__dict__[key])
except ValueError as va_ex:
fmt_record_item += "{}: '{}'".format(
type(record.__dict__[key]).__name__,
str(record.__dict__[key]))
hash = "{0:03d}".format(meta['order'])
fmt_record_items[hash] = fmt_record_item
str_fmt_record = ""
for key in sorted(fmt_record_items.keys()):
value = fmt_record_items[key]
str_fmt_record += value
return str_fmt_record[:-2]
def _error_message(error_message, error_code):
error_message_ = None
exit_code_description_ = requests_fortified_error_desc(error_code).rstrip('\.')
error_message_prefix_ = "{0}: {1}".format(error_code, exit_code_description_)
error_message = safe_str(error_message).strip()
if error_message:
error_message_ = "%s: '%s'" % (error_message_prefix_, six.text_type(error_message))
elif exit_code_description_:
error_message_ = error_message_prefix_
return error_message_
def _error_message(error_message, error_code):
error_message_ = None
exit_code_description_ = tune_request_error_desc(error_code).rstrip(
'\.')
error_message_prefix_ = f"{error_code}: {exit_code_description_}"
error_message = safe_str(error_message).strip()
if error_message:
error_message_ = "%s: '%s'" % (error_message_prefix_,
six.text_type(error_message))
elif exit_code_description_:
error_message_ = error_message_prefix_
return error_message_
def _error_message(error_message, error_code):
error_message_ = None
exit_code_description_ = requests_fortified_error_desc(
error_code).rstrip('\.')
error_message_prefix_ = "{0}: {1}".format(error_code,
exit_code_description_)
error_message = safe_str(error_message).strip()
if error_message:
error_message_ = "%s: '%s'" % (error_message_prefix_,
six.text_type(error_message))
elif exit_code_description_:
error_message_ = error_message_prefix_
return error_message_
def tune_v2_request_retry_func(self, response):
"""Request Retry Function
Args:
response:
Returns:
Boolean
"""
try:
response_json = response.json()
except Exception as ex:
# No JSON response available.
raise TuneReportingError(
error_message='Invalid JSON response: {}'.format(response.text),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_JSON_DECODING
)
self.logger.debug("TMC API V2: Check for Retry: Start", extra=response_json)
tune_v2_status_code = None
tune_v2_errors_messages = ""
if 'status_code' in response_json:
tune_v2_status_code = response_json['status_code']
if 'errors' in response_json:
tune_v2_errors = response_json['errors']
if isinstance(tune_v2_errors, dict) \
and 'message' in tune_v2_errors:
tune_v2_errors_messages += tune_v2_errors['message']
elif isinstance(tune_v2_errors, list):
for tune_v2_error in tune_v2_errors:
if isinstance(tune_v2_error, dict) \
and 'message' in tune_v2_error:
tune_v2_errors_messages += tune_v2_error['message']
tune_v2_status_type = get_http_status_type(tune_v2_status_code)
response_extra = {
'status_code': tune_v2_status_code,
'status_type': tune_v2_status_type,
}
if tune_v2_errors_messages:
response_extra.update({'error_messages': safe_str(tune_v2_errors_messages)})
self.logger.debug("TMC API Base: Check for Retry: Response", extra=response_extra)
if tune_v2_status_code == 200:
self.logger.debug("TMC API Base: Check for Retry: Success", extra=response_extra)
return False
if tune_v2_status_code in [401, 403]:
self.logger.error("TMC API: Request: Error", extra={'status_code': tune_v2_status_code})
raise TuneRequestError(
error_message=tune_v2_errors_messages,
error_code=tune_v2_status_code,
)
if tune_v2_status_code in [404, 500]:
if "Api key was not found." in tune_v2_errors_messages:
self.logger.error("TMC API: Request: Error", extra={'tune_v2_status_code': tune_v2_status_code})
raise TuneRequestError(
error_message=tune_v2_errors_messages,
error_code=tune_v2_status_code,
)
self.logger.warning("TMC API Base: Check for Retry: Retry Candidate", extra=response_extra)
return True
self.logger.warning("TMC API Base: Check for Retry: No Retry", extra=response_extra)
return False
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
request_cert=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
request_cert: (optional) Cert tuple to enable Client side certificates.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise RequestsFortifiedValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise RequestsFortifiedValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
"{0}: Session: Details".format(request_label),
extra={
'cookie_payload': self.requests_session_client.session.cookies.get_dict(),
'request_label': request_label
}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug("{0}: Details".format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if request_cert:
kwargs.update({'cert': request_cert})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.note(
"{0}: Curl".format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug(
"{0}: {1}".format(request_label, request_method),
extra={'response_url': response.url}
)
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.requests_session_client.request(**kwargs)
except Exception as ex:
self.logger.error(
"{0}: Request Base: Error".format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
"{0}: Response: Failed".format(request_label),
extra={
'request_curl': self.built_request_curl
},
)
raise RequestsFortifiedModuleError(
error_message="{0}: Response: Failed".format(request_label),
error_code=RequestsFortifiedErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
"{0}: Response: Details".format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
"{0}: Cookie Payload".format(request_label),
extra={
'cookie_payload': self.requests_session_client.session.cookies.get_dict(),
'request_label': request_label
}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error("{0}: Response: Failed".format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error("{0}: Error: Response: Details".format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise RequestsFortifiedClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise RequestsFortifiedServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error("{0}: Error: Unhandled".format(request_label), extra=extra_unhandled)
raise RequestsFortifiedModuleError(**kwargs)
def main(tmc_api_key):
TIMEZONE_COLLECT = "America/New_York"
tune_v2_advertiser_stats_actuals = \
TuneV2AdvertiserStatsActuals(
timezone=TIMEZONE_COLLECT,
logger_level=logging.INFO,
logger_format=TuneLoggingFormat.JSON
)
dw_file_path = "data.{}".format(TuneV2AdvertiserStatsFormats.JSON)
if os.path.exists(dw_file_path):
os.remove(dw_file_path)
tz = pytz.timezone(TIMEZONE_COLLECT)
yesterday = datetime.now(tz).date() - timedelta(days=1)
str_yesterday = str(yesterday)
request_params = {
'format':
TuneV2AdvertiserStatsFormats.CSV,
'fields': ("ad_clicks,"
"ad_clicks_unique,"
"ad_impressions,"
"ad_impressions_unique,"
"ad_network_id,"
"advertiser_id,"
"country.code,"
"date_hour,"
"events,"
"installs,"
"is_reengagement,"
"payouts,"
"publisher_id,"
"publisher_sub_ad.ref,"
"publisher_sub_adgroup.ref,"
"publisher_sub_campaign.ref,"
"publisher_sub_publisher.ref,"
"publisher_sub_site.ref,"
"site_id"),
'group': ("country_id,"
"is_reengagement,"
"publisher_id,"
"publisher_sub_ad_id,"
"publisher_sub_adgroup_id,"
"publisher_sub_campaign_id,"
"publisher_sub_publisher_id,"
"publisher_sub_site_id,"
"site_id"),
'timezone':
"America/Los_Angeles"
}
try:
tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key)
response = tune_v2_advertiser_stats_actuals.stream(
auth_value=tmc_api_key,
auth_type=TuneV2AuthenticationTypes.API_KEY,
auth_type_use=TuneV2AuthenticationTypes.API_KEY,
start_date=str_yesterday,
end_date=str_yesterday,
request_params=request_params,
request_retry={
'delay': 15,
'timeout': 30,
'tries': 5
})
line_count = 0
csv_keys_list = None
json_keys_dict = {
"publisher_sub_campaign.ref": "sub_campaign_ref",
"publisher_sub_ad.ref": "sub_ad_ref",
"publisher_sub_adgroup.ref": "sub_adgroup_ref",
"publisher_sub_publisher.ref": "sub_publisher_ref",
"publisher_sub_site.ref": "sub_site_ref",
"publisher_sub_placement.ref": "sub_placement_ref",
"publisher_sub_campaign.name": "sub_campaign_name",
"publisher_sub_ad.name": "sub_ad_name",
"publisher_sub_adgroup.name": "sub_adgroup_name",
"publisher_sub_publisher.name": "sub_publisher_name",
"publisher_sub_site.name": "sub_site_name",
"publisher_sub_placement.name": "sub_placement_name",
"publisher_sub_campaign_id": "sub_campaign_id",
"publisher_sub_ad_id": "sub_ad_id",
"publisher_sub_adgroup_id": "sub_adgroup_id",
"publisher_sub_publisher_id": "sub_publisher_id",
"publisher_sub_site_id": "sub_site_id",
"publisher_sub_placement_id": "publisher_sub_placement_id",
"country.code": "country_code",
"ad_impressions": "received_impressions_gross",
"ad_impressions_unique": "received_impressions_unique",
"ad_clicks": "received_clicks_gross",
"ad_clicks_unique": "received_clicks_unique",
"events": "received_engagements",
"installs": "received_installs",
"payouts": "cost"
}
json_types_dict = {
"client_id": int,
"partner_id": int,
"vendor_id": int,
"date": str,
"hour": int,
"timezone": str,
"granularity": str,
"site_ref_id": str,
"site_ref_type": str,
"partner_ref_id": int,
"partner_ref_type": str,
"partner_vendor_ref_id": int,
"partner_vendor_ref_type": str,
"sub_campaign_type": str,
"sub_campaign_ref": str,
"sub_ad_ref": str,
"sub_adgroup_ref": str,
"sub_publisher_ref": str,
"sub_site_ref": str,
"sub_placement_ref": str,
"sub_campaign_name": str,
"sub_ad_name": str,
"sub_adgroup_name": str,
"sub_publisher_name": str,
"sub_site_name": str,
"sub_placement_name": str,
"sub_campaign_name": str,
"sub_ad_name": str,
"sub_adgroup_name": str,
"sub_publisher_name": str,
"sub_site_name": str,
"sub_placement_name": str,
"sub_campaign_id": int,
"sub_ad_id": int,
"sub_adgroup_id": int,
"sub_publisher_id": int,
"sub_site_id": int,
"publisher_sub_placement_id": int,
"country_code": str,
"received_impressions_gross": int,
"received_impressions_unique": int,
"received_clicks_gross": int,
"received_clicks_unique": int,
"received_installs": int,
"received_engagements": int,
"received_conversions": int,
"cost": float,
"cost_currency": str,
"site_id": int,
"publisher_id": int,
"advertiser_id": int,
"ad_network_id": int,
"ad_impressions": int,
}
client_id = 0
partner_id = 0
vendor_id = 0
timezone = "TBD"
granularity = "TBD"
config_extra = {
"client_id": client_id,
"partner_id": partner_id,
"vendor_id": vendor_id,
"timezone": timezone,
"granularity": granularity,
"cost_currency": "USD",
"received_conversions": 0,
"site_ref_type": "tmc",
"partner_ref_type": "tmc",
"partner_vendor_ref_type": "tmc"
}
with open(file=dw_file_path, mode='w') as dw_file_w:
for bytes_line in response.iter_lines(chunk_size=4096):
if bytes_line: # filter out keep-alive new chunks
line_count += 1
str_line = bytes_line.decode("utf-8")
if line_count == 1:
csv_keys_list = str_line.split(',')
for index, csv_key in enumerate(csv_keys_list):
if csv_key in json_keys_dict:
csv_keys_list[index] = json_keys_dict[csv_key]
continue
elif line_count > 2:
dw_file_w.write('\n')
csv_values_list = str_line.split(',')
json__dict = {}
is_reengagement = 0
received_installs = 0
received_engagements = 0
for csv_key, csv_value in zip(csv_keys_list,
csv_values_list):
csv_value_strip = csv_value.strip('"')
if csv_key == "date_hour":
parts_date_time = csv_value_strip.split(" ")
rdate_yyyy_mm_dd = parts_date_time[0]
parts_time = parts_date_time[1].split(":")
rhour = safe_int(parts_time[0])
json__dict.update({"date": rdate_yyyy_mm_dd})
json__dict.update({"hour": rhour})
elif csv_key == "is_reengagement":
is_reengagement = safe_int(csv_value_strip)
elif csv_key == "received_installs":
received_installs = safe_int(csv_value_strip)
json__dict.update(
{'received_installs': received_installs})
elif csv_key == "received_engagements":
received_engagements = safe_int(csv_value_strip)
json__dict.update(
{'received_engagements': received_engagements})
else:
if csv_key in json_types_dict:
if json_types_dict[csv_key] == str:
csv_value_typed = safe_str(csv_value_strip)
elif json_types_dict[csv_key] == int:
csv_value_typed = safe_int(csv_value_strip)
elif json_types_dict[csv_key] == float:
csv_value_typed = safe_float(
csv_value_strip)
else:
csv_value_typed = safe_str(csv_value_strip)
else:
csv_value_typed = safe_str(csv_value_strip)
json__dict.update({csv_key: csv_value_typed})
if is_reengagement == 1:
engagements = received_engagements
else:
engagements = 0
if engagements > 0 and received_installs > 0:
sub_campaign_type = "acquisition_engagement"
elif received_installs > 0:
sub_campaign_type = "acquisition"
elif engagements > 0:
sub_campaign_type = "engagement"
else:
sub_campaign_type = ""
json__dict.update({'sub_campaign_type': sub_campaign_type})
json__dict.update(config_extra)
json_str = json.dumps(json__dict)
dw_file_w.write(json_str)
dw_file_w.flush()
statinfo = os.stat(dw_file_path)
extra = {
'response_status_code': response.status_code,
'response_headers': response.headers,
'dw_file_path': dw_file_path,
'dw_file_size': convert_size(statinfo.st_size),
'line_count': line_count,
'csv_header_list': csv_keys_list
}
pprint(extra)
except TuneRequestBaseError as tmc_req_ex:
print_traceback(tmc_req_ex)
pprint(tmc_req_ex.to_dict())
print(str(tmc_req_ex))
except TuneReportingError as tmc_rep_ex:
pprint(tmc_rep_ex.to_dict())
print(str(tmc_rep_ex))
except Exception as ex:
print_traceback(ex)
print(get_exception_message(ex))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @copyright 2017 TUNE, Inc. (http://www.tune.com)
# @namespace smart-cast
from pprintpp import pprint
from safe_cast import (
safe_int,
safe_float,
safe_str,
)
pprint(safe_int(4))
pprint(safe_float(4))
pprint(safe_str(4))
pprint(safe_int('4'))
pprint(safe_float('4'))
pprint(safe_str('4'))
pprint(safe_int(4.0))
pprint(safe_float(4.0))
pprint(safe_str(4.0))
pprint(safe_int('4.0'))
pprint(safe_float('4.0'))
pprint(safe_str('4.0'))
pprint(safe_int('1.0'))
pprint(safe_int('1'))
def validate_response(
response,
request_curl,
request_label=None,
):
"""Validate response
Args:
response:
request_curl:
request_label:
Returns:
"""
response_extra = {}
if request_label is None:
request_label = 'Validate Response'
if not response:
error_message = f'{request_label}: Failed: None'
log.error(error_message, extra=response_extra)
raise TuneRequestModuleError(
error_message=error_message,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
log.debug(f'{request_label}: Defined', extra=response_extra)
response_extra.update({'http_status_code': response.status_code})
# Not using hasattr on purpose
# Assuming positive approach will give us speedup since when attribute exists
# hasattr takes double the time.
# Anyway we use text attribute here to logging purpose only
try:
response_extra.update({'response_text_length': len(response.text)})
except AttributeError:
pass
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_extra.update(
{'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_extra.update({
'Content-Length':
bytes_to_human(response_headers_content_length)
})
if 'Content-Encoding' in response.headers:
response_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_extra.update(
{'Content-Encoding': response_content_encoding})
if 'Transfer-Encoding' in response.headers:
response_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_extra.update(
{'Transfer-Encoding': response_transfer_encoding})
if not is_http_status_successful(http_status_code=response.status_code):
error_message = f'{request_label}: Failed'
log.error(error_message, extra=response_extra)
raise TuneRequestModuleError(
error_message=error_message,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
log.debug(f'{request_label}: Success', extra=response_extra)
def build_response_error_details(request_label, request_url, response):
"""Build gather status of Requests' response.
Args:
request_label:
request_url:
response:
Returns:
"""
http_status_code = \
response.status_code
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_status = "%s: %s: %s" % (http_status_code, http_status_type, http_status_desc)
response_error_details = {
'request_url': request_url,
'request_label': request_label,
'response_status': response_status,
'response_status_code': http_status_code,
'response_status_type': http_status_type,
'response_status_desc': http_status_desc
}
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_error_details.update({'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers and \
response.headers['Content-Length']:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_error_details.update({'Content-Length': response_headers_content_length})
if 'Transfer-Encoding' in response.headers and \
response.headers['Transfer-Encoding']:
response_headers_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_error_details.update({'Transfer-Encoding': response_headers_transfer_encoding})
if 'Content-Encoding' in response.headers and \
response.headers['Content-Encoding']:
response_headers_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_error_details.update({'Content-Encoding': response_headers_content_encoding})
if hasattr(response, "reason") and response.reason:
response_error_details.update({'response_reason': response.reason})
response_details = None
response_details_source = None
try:
response_details = response.json()
response_details_source = 'json'
except Exception:
if hasattr(response, 'text') and \
response.text and \
len(response.text) > 0:
response_details = response.text
response_details_source = 'text'
if response_details.startswith('<html'):
response_details_source = 'html'
soup_html = bs4.BeautifulSoup(response_details, "html.parser")
# kill all script and style elements
for script in soup_html(["script", "style"]):
script.extract() # rip it out
text_html = soup_html.get_text()
lines_html = [line for line in text_html.split('\n') if line.strip() != '']
lines_html = [line.strip(' ') for line in lines_html]
response_details = lines_html
elif response_details.startswith('<?xml'):
response_details_source = 'xml'
response_details = json.dumps(xmltodict.parse(response_details))
response_error_details.update({
'response_details': response_details,
'response_details_source': response_details_source
})
# pprint(response_error_details)
return response_error_details
def test_safe_str():
# test type:
assert isinstance(safe_str(0), str)
assert isinstance(safe_str(0.0), str)
assert isinstance(safe_str('0'), str)
# test str cast:
assert safe_str('') == ''
assert safe_str('Hello Jeff') == 'Hello Jeff'
assert safe_str('@%#$%^&*') == '@%#$%^&*'
assert safe_str(None) == ''
# test numeric cast:
assert safe_str(0) == '0'
assert safe_str(10) == '10'
assert safe_str(-1) == '-1'
assert safe_str(10.52) == '10.52'
assert safe_str(-1.32) == '-1.32'
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @copyright 2018 TUNE, Inc. (http://www.tune.com)
# @namespace smart-cast
from safe_cast import (safe_int, safe_float, safe_str, safe_dict, safe_cost,
safe_fraction)
print("\n")
print('==================================')
print('safe_str()')
print('==================================')
print(safe_str(.1))
print(safe_str(.12))
print(safe_str(.123))
print(safe_str(.1234))
print(safe_str(.12345))
print(safe_str(4))
print(safe_str('4'))
print(safe_str('4.0'))
print(safe_str(4.0))
print("\n")
print('==================================')
print('safe_int()')
print('==================================')
print(safe_int(.1))
print(safe_int(.12))
print(safe_int(.123))
print(safe_int(.1234))
print(safe_int(.12345))
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise TuneRequestValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise TuneRequestValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
'{}: Session: Details'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug('{}: Details'.format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.debug(
'{}: Curl'.format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url})
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.tune_request.request(**kwargs)
except Exception as ex:
self.logger.error(
'{}: Request Base: Error'.format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
'{}: Response: Failed'.format(request_label),
extra={'request_curl': self.built_request_curl},
)
raise TuneRequestModuleError(
error_message='{}: Response: Failed'.format(request_label),
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
http_status_code_to_type(http_status_code)
http_status_desc = \
http_status_code_to_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
'{}: Response: Details'.format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
'{}: Cookie Payload'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled)
raise TuneRequestModuleError(**kwargs)
def validate_response(
response,
request_curl,
request_label=None,
):
"""Validate response
Args:
response:
request_curl:
request_label:
Returns:
"""
response_extra = {}
if request_label is None:
request_label = 'Validate Response'
if not response:
error_message = "{0}: Failed: None".format(request_label)
log.error(error_message, extra=response_extra)
raise RequestsFortifiedModuleError(
error_message=error_message,
error_request_curl=request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_SOFTWARE
)
log.debug("{0}: Defined".format(request_label), extra=response_extra)
response_extra.update({'http_status_code': response.status_code})
# Not using hasattr on purpose
# Assuming positive approach will give us speedup since when attribute exists
# hasattr takes double the time.
# Anyway we use text attribute here to logging purpose only
try:
response_extra.update({'response_text_length': len(response.text)})
except AttributeError:
pass
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_extra.update({'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_extra.update({'Content-Length': bytes_to_human(response_headers_content_length)})
if 'Content-Encoding' in response.headers:
response_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_extra.update({'Content-Encoding': response_content_encoding})
if 'Transfer-Encoding' in response.headers:
response_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_extra.update({'Transfer-Encoding': response_transfer_encoding})
if not is_http_status_successful(http_status_code=response.status_code):
error_message = "{0}: Failed".format(request_label)
log.error(error_message, extra=response_extra)
raise RequestsFortifiedModuleError(
error_message=error_message,
error_request_curl=request_curl,
error_code=RequestsFortifiedErrorCodes.REQ_ERR_SOFTWARE
)
log.debug("{0}: Success".format(request_label), extra=response_extra)
def build_response_error_details(request_label, request_url, response):
"""Build gather status of Requests' response.
Args:
request_label:
request_url:
response:
Returns:
"""
http_status_code = \
response.status_code
http_status_type = \
get_http_status_type(http_status_code)
http_status_desc = \
get_http_status_desc(http_status_code)
response_status = f"{http_status_code}: {http_status_type}: {http_status_desc}"
response_error_details = {
'request_url': request_url,
'request_label': request_label,
'response_status': response_status,
'response_status_code': http_status_code,
'response_status_type': http_status_type,
'response_status_desc': http_status_desc
}
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_error_details.update(
{'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers and \
response.headers['Content-Length']:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_error_details.update(
{'Content-Length': response_headers_content_length})
if 'Transfer-Encoding' in response.headers and \
response.headers['Transfer-Encoding']:
response_headers_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_error_details.update(
{'Transfer-Encoding': response_headers_transfer_encoding})
if 'Content-Encoding' in response.headers and \
response.headers['Content-Encoding']:
response_headers_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_error_details.update(
{'Content-Encoding': response_headers_content_encoding})
if hasattr(response, "reason") and response.reason:
response_error_details.update({'response_reason': response.reason})
response_details = None
response_details_source = None
try:
response_details = response.json()
response_details_source = 'json'
except Exception:
if hasattr(response, 'text') and \
response.text and \
len(response.text) > 0:
response_details = response.text
response_details_source = 'text'
if response_details.startswith('<html'):
response_details_source = 'html'
soup_html = bs4.BeautifulSoup(response_details, "html.parser")
# kill all script and style elements
for script in soup_html(["script", "style"]):
script.extract() # rip it out
text_html = soup_html.get_text()
lines_html = [
line for line in text_html.split('\n')
if line.strip() != ''
]
lines_html = [line.strip(' ') for line in lines_html]
response_details = lines_html
elif response_details.startswith('<?xml'):
response_details_source = 'xml'
response_details = json.dumps(
xmltodict.parse(response_details))
response_error_details.update({
'response_details':
response_details,
'response_details_source':
response_details_source
})
# pprint(response_error_details)
return response_error_details
