def _read_stats_data(self, export_job_handle, report_url, report_format=None):
if not export_job_handle:
raise ValueError("Argument 'export_job_handle' not assigned.")
if not report_url:
raise ValueError("Argument 'report_url' not assigned.")
self.logger.info(("TMC v3 Logs Advertisers Base: "
"Start Advertiser Report Stats read: "
"Job Handle: '{}'").format(export_job_handle))
if not report_format:
path = urlparse(report_url).path
ext = os.path.splitext(path)[1]
report_format = ext[1:]
self.logger.info(("TMC v3 Logs Advertisers Base: "
"Reading URL '{}' with format '{}'").format(report_url, report_format))
# pylint: disable=redefined-variable-type
if report_format == TuneV3LogsAdvertisersResponseFormats.JSON:
report_reader = ReportReaderJSON(report_url)
elif report_format == TuneV3LogsAdvertisersResponseFormats.CSV:
report_reader = ReportReaderCSV(report_url)
else:
raise TuneReportingError(error_message="Unexpected report format: '{}'".format(report_format))
# pylint: enable=redefined-variable-type
if not report_reader:
raise TuneReportingError(error_message="Report reader not created for format {}".format(report_format))
report_reader.read()
self.logger.info(("TMC v3 Logs Advertisers Base: " "Finished Advertiser Report Stats read"))
return report_reader.generator
def _read_stats_data(self, export_job_id, report_url, report_format=None):
"""Read v2 Advertiser Stats Data
Args:
export_job_id:
report_url:
report_format:
Returns:
"""
if not export_job_id:
raise ValueError("Argument 'job_id' not assigned.")
if not report_url:
raise ValueError("Argument 'report_url' not assigned.")
self.logger.info(msg=("TMC v2 Advertiser Stats: "
"Start Advertiser Report Stats Read"),
extra={'job_id': export_job_id})
if not report_format:
path = urlparse(report_url).path
ext = os.path.splitext(path)[1]
report_format = ext[1:]
self.logger.info("TMC v2 Advertiser Stats: Reading",
extra={
'report_url': report_url,
'report_format': report_format
})
# pylint: disable=redefined-variable-type
if report_format == TuneV2AdvertiserStatsFormats.JSON:
report_reader = ReportReaderJSON(report_url)
elif report_format == TuneV2AdvertiserStatsFormats.CSV:
report_reader = ReportReaderCSV(report_url)
else:
raise TuneReportingError(
error_message=(
"Unexpected Report format: '{}'").format(report_format),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
# pylint: enable=redefined-variable-type
if not report_reader:
raise TuneReportingError(
error_message=("Report reader not created for format {}"
).format(report_format),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
report_reader.read()
self.logger.info(("TMC v2 Advertiser Stats: "
"Finished Advertiser Report Stats read"))
return report_reader.generator
def collect(
self,
auth_value,
auth_type,
auth_type_use,
start_date,
end_date,
advertiser_id,
request_params=None,
request_retry=None,
request_action=TuneV3LogsAdvertisersActions.FIND
):
"""Collect data: TUNE Advertiser Stats Actuals.
Args:
tmc_api_key:
start_date:
end_date:
advertiser_id:
request_params:
request_retry:
request_action:
Returns:
"""
if not auth_value:
raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'auth_value' not provided.")
if not auth_type:
raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'auth_type' not valid.")
if not auth_type_use or \
not TuneV2AuthenticationTypes.validate(auth_type_use):
raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'auth_type_use' not valid.")
if not advertiser_id:
raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'advertiser_id' not valid.")
if auth_type == TuneV2AuthenticationTypes.API_KEY:
self.api_key = auth_value
if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = tmc_auth_v2_session_token(
tmc_api_key=self.api_key, logger_level=self.logger_level, logger_format=self.logger_format
)
elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = auth_value
else:
raise ValueError(error_message="Invalid 'auth_type': '{}'".format(auth_type))
auth_value_use = None
if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
if self.session_token is None:
raise TuneReportingError(
error_message="Value 'session_token' not defined.",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE
)
auth_value_use = self.session_token
elif auth_type_use == TuneV2AuthenticationTypes.API_KEY:
if self.api_key is None:
raise TuneReportingError(
error_message="Value 'api_key' not defined.",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE,
)
auth_value_use = self.api_key
if not auth_value_use:
raise TuneReportingError(
error_message="Value 'auth_value_use' not defined.",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE,
)
self.advertiser_id = advertiser_id
self.controller = self._CONTROLLER.format(advertiser_id=self.advertiser_id)
self._collect(auth_type_use, start_date, end_date, request_params, request_retry, request_action)
def _check_v2_job_status_on_queue(
self,
auth_type,
auth_value,
export_status_controller,
export_status_action,
export_job_id,
request_retry=None,
):
"""Check Job Export Status
Args:
export_status_controller:
export_status_action:
export_job_id:
request_retry:
Returns:
"""
request_label = "TMC v2 Advertiser Stats: Check Export Status"
v2_export_status_request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=export_status_controller,
action=export_status_action
)
request_params = {auth_type: auth_value, "job_id": export_job_id}
self.logger.info(
("TMC v2 Advertiser Stats: Check Job Status"),
extra={
'action': export_status_action,
'job_id': export_job_id,
'request_url': v2_export_status_request_url,
'request_params': safe_dict(request_params)
})
tries = 60 # -1 (indefinite)
delay = 10
jitter = 10
max_delay = 60
if request_retry is not None:
if 'delay' in request_retry:
delay = request_retry['delay']
if 'jitter' in request_retry:
jitter = request_retry['jitter']
if 'max_delay' in request_retry:
max_delay = request_retry['max_delay']
if 'tries' in request_retry:
tries = request_retry['tries']
else:
request_retry.update({'tries': 60})
else:
request_retry = {'tries': 60, 'delay': 10, 'timeout': 60}
self.logger.debug(msg=("TMC v2 Advertiser Stats: Check Job Status: "
"Request Retry"),
extra=request_retry)
report_url = None
_attempts = 1
export_percent_complete = 0
time.sleep(10)
_tries, _delay = tries, delay
while True:
try:
response = self.mv_request.request(
request_method="GET",
request_url=v2_export_status_request_url,
request_params=request_params,
request_label=request_label,
request_retry_func=self.tune_v2_request_retry_func)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: {}".format(
get_exception_message(ex)))
raise
http_status_successful = is_http_status_type(
http_status_code=response.status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
if not http_status_successful:
raise TuneReportingError(
error_message=(
"Failed to get export status on queue: {}").format(
response.status_code),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
if hasattr(response, 'url'):
self.logger.info(
"TMC v2 Advertiser Stats: Reporting API: Status URL",
extra={'response_url': response.url})
json_response = response.json()
if not json_response:
request_status_successful = False
elif 'status_code' not in json_response:
request_status_successful = False
else:
status_code = json_response['status_code']
request_status_successful = is_http_status_type(
http_status_code=status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
errors = None
if 'errors' in json_response:
errors = json_response['errors']
if not request_status_successful:
error_message = (
"TMC v2 Advertiser Stats: Check Job Status: GET '{}', Failed: {}, {}"
).format(v2_export_status_request_url, status_code, errors)
if (status_code ==
TuneReportingError.EX_SRV_ERR_500_INTERNAL_SERVER):
self.logger.error(error_message)
elif (status_code ==
TuneReportingError.EX_SRV_ERR_503_SERVICE_UNAVAILABLE):
self.logger.error(error_message)
elif (status_code ==
TuneReportingError.EX_SRV_ERR_504_SERVICE_TIMEOUT):
self.logger.error(error_message)
continue
elif (status_code ==
TuneReportingError.EX_CLT_ERR_408_REQUEST_TIMEOUT):
self.logger.error(
"GET '{}' request timeout, Retrying: {}".format(
v2_export_status_request_url, status_code))
continue
else:
raise TuneReportingError(error_message=error_message,
error_code=status_code)
if tries >= 0 and _tries <= 1:
if (status_code == HttpStatusCode.GATEWAY_TIMEOUT):
raise TuneReportingError(
error_message=error_message,
error_code=TuneReportingErrorCodes.GATEWAY_TIMEOUT)
elif (status_code == HttpStatusCode.REQUEST_TIMEOUT):
raise TuneReportingError(
error_message=error_message,
error_code=TuneReportingErrorCodes.REQUEST_TIMEOUT)
else:
raise TuneReportingError(error_message=error_message,
error_code=status_code)
else:
self.logger.warning(error_message)
export_percent_complete = 0
if 'data' in json_response and json_response['data']:
json_data = json_response['data']
if "percent_complete" in json_data:
export_percent_complete = \
safe_int(json_data["percent_complete"])
self.logger.info(msg=("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"Response Success"),
extra={
'job_id':
export_job_id,
'export_status':
json_data["status"],
'export_percent_complete':
safe_int(export_percent_complete),
'attempt':
_attempts
})
if (export_status_action ==
TuneV2AdvertiserStatsStatusAction.STATUS):
if (export_percent_complete == 100
and json_data["status"] == "complete"
and json_data["url"]):
report_url = json_data["url"]
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Completed"),
extra={
'job_id': export_job_id,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label
})
break
elif (export_status_action ==
TuneV2AdvertiserStatsStatusAction.DOWNLOAD):
if (export_percent_complete == 100
and json_data["status"] == "complete"
and json_data["data"]["url"]):
report_url = json_data["data"]["url"]
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Completed"),
extra={
'job_id': export_job_id,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label
})
break
else:
self.logger.debug("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"No Data Available")
if tries >= 0:
_tries -= 1
if _tries == 0:
self.logger.error(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Exhausted Retries"),
extra={
'attempt':
_attempts,
'tries':
_tries,
'action':
export_status_action,
'request_label':
request_label,
'export_percent_complete':
safe_int(export_percent_complete),
'job_id':
export_job_id
})
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"Exhausted Retries: "
"Percent Completed: {}").format(
safe_int(export_percent_complete)),
error_code=TuneReportingErrorCodes.
REP_ERR_RETRY_EXHAUSTED)
_attempts += 1
self.logger.info("TMC v2 Advertiser Stats: Check Job Status",
extra={
'attempt': _attempts,
'job_id': export_job_id,
'delay': _delay,
'action': export_status_action
})
time.sleep(_delay)
_delay += jitter
_delay = min(_delay, max_delay)
if export_percent_complete == 100 and not report_url:
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: Check Job Export Status: "
"Download report URL: Undefined"),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
self.logger.info(
"TMC v2 Advertiser Stats: Check Job Export Status: Finished",
extra={
'attempt': _attempts,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': export_percent_complete,
'job_id': export_job_id
})
return report_url
def _export_v2_job_to_queue(
self,
export_controller,
export_action,
request_params,
request_retry=None,
request_label="TMC v2 Job To Queue",
):
"""Export Report Request to Job Queue
Args:
export_controller:
export_action:
request_params:
request_retry:
Returns:
Export Job ID
"""
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=export_controller,
action=export_action
)
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Start Advertiser Report Actuals"),
extra={
'action': export_action,
'start_date': request_params["start_date"],
'end_date': request_params["end_date"]
})
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_label=request_label,
request_retry_func=self.tune_v2_request_retry_func)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Request Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Reporting Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(get_exception_message(ex))
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: Unexpected Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Advertiser Stats: Action '{}'".format(
export_action))
if hasattr(response, 'url'):
self.logger.info(
"TMC v2 Advertiser Stats: Reporting API: Export URL",
extra={'response_url': response.url})
if (not json_response or json_response['status_code'] != 200
or 'errors' in json_response):
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Action '{}': "
"Failed to export stats: {}, {}").format(
export_action, json_response['status_code'],
json_response['errors']),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
if ('data' not in json_response or not json_response['data']):
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Action '{}': "
"Missing data").format(export_action),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
json_data = json_response['data']
export_job_id = None
if export_action == TuneV2AdvertiserStatsExportActions.EXPORT:
if ('job_id' not in json_data or not json_data['job_id']):
raise TuneReportingError(error_message=(
"TMC v2 Advertiser Stats: "
"Action '{}': "
"Response missing 'export_job_id': {}").format(
export_action, str(json_data)),
error_code=TuneReportingErrorCodes.
REP_ERR_UNEXPECTED_VALUE)
export_job_id = json_data['job_id']
elif export_action == TuneV2AdvertiserStatsExportActions.FIND_EXPORT_QUEUE:
export_job_id = json_data
if not export_job_id:
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: "
"Action '{}': "
"Response missing 'job_id'").format(export_action),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
self.logger.info("TMC v2 Advertiser Stats: Reporting API: Job ID",
extra={'job_id': export_job_id})
return export_job_id
def _process_export_stream_v2(
self,
auth_type_use,
str_date_start,
str_date_end,
export_controller,
export_action,
export_status_controller,
export_status_action,
request_params,
request_retry,
request_label="TMC v2 Advertiser Stats Export Stream",
):
"""Process Export Job by Steaming
Args:
str_date_start:
str_date_end:
export_controller:
export_action:
export_status_controller:
export_status_action:
request_params:
request_retry:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
str_date_start += " 00:00:00"
str_date_end += " 23:59:59"
request_params["start_date"] = \
str_date_start
request_params["end_date"] = \
str_date_end
self.logger.debug(
"TMC v2 Advertiser Stats: Export Stream V2: Export Job to Queue",
extra={
'controller': export_controller,
'action': export_action,
'request_params': safe_dict(request_params),
'request_retry': safe_dict(request_retry)
})
export_job_id = self._export_v2_job_to_queue(
export_controller,
export_action,
request_params,
request_retry,
)
self.logger.debug(
"TMC v2 Advertiser Stats: Export Stream V2: Check Job status on Queue",
extra={
'controller': export_status_controller,
'action': export_status_action,
'job_id': export_job_id
})
export_report_url = self._check_v2_job_status_on_queue(
auth_type_use,
auth_value,
export_status_controller,
export_status_action,
export_job_id,
request_retry=request_retry)
if not export_report_url:
raise TuneReportingError(
error_message="Export URL not defined",
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE,
)
self.logger.info(("TMC v2 Advertiser Stats: "
"Export Stream V2: Request Completed Job"),
extra={
'job_id': export_job_id,
'report_url': export_report_url
})
response = self.mv_request.request(request_method="GET",
request_url=export_report_url,
stream=True,
request_label=request_label)
self.logger.info(
"TMC v2 Advertiser Stats: Export Stream V2: Response Completed Job",
extra={
'response_status_code': response.status_code,
'response_headers': response.headers,
'job_id': export_job_id,
'report_url': export_report_url
})
return response
def _process_export_download_csv_v2(
self,
tmp_directory,
data,
auth_type_use,
str_date_start,
str_date_end,
export_controller,
export_action,
export_status_controller,
export_status_action,
request_params,
request_retry,
):
"""Process Export Job by Reading Downloaded CSV
Args:
data:
str_date_start:
str_date_end:
export_controller:
export_action:
export_status_controller:
export_status_action:
request_params:
request_retry:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
str_date_start += " 00:00:00"
str_date_end += " 23:59:59"
request_params["start_date"] = \
str_date_start
request_params["end_date"] = \
str_date_end
export_job_id = self._export_v2_job_to_queue(
export_controller,
export_action,
request_params,
request_retry,
)
export_report_url = self._check_v2_job_status_on_queue(
auth_type_use,
auth_value,
export_status_controller,
export_status_action,
export_job_id,
request_retry=request_retry,
)
if not export_report_url:
raise TuneReportingError(
error_message="Export URL not defined",
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE,
)
self.logger.info("TuneV2AdvertiserStatsBase",
extra={
'job_id': export_job_id,
'report_url': export_report_url
})
tmp_csv_file_name = self._TUNE_ADVERTISING_STATS_TPL.format(
job_id=export_job_id, format=TuneV2AdvertiserStatsFormats.CSV)
tmp_csv_file_name = tmp_csv_file_name.replace('-', '_')
job_row_count = 0
for row in self.mv_request.request_csv_download(
tmp_csv_file_name=tmp_csv_file_name,
tmp_directory=tmp_directory,
request_method="GET",
request_url=export_report_url,
request_label="TMC v2 Advertiser Stats: Export: Download CSV"):
# for row in list(generator_data_stats):
if len(row) == 0:
continue
job_row_count += 1
data.append(row)
if os.path.exists(tmp_csv_file_name):
os.remove(tmp_csv_file_name)
self.logger.debug(
"TMC v2 Advertiser Stats: Process Export Job by Reading Downloaded CSV",
extra={
'action': 'export',
'start_date': str_date_start,
'end_date': str_date_end,
'job_id': export_job_id,
'row_count': job_row_count
})
return (data, job_row_count)
def _map_request_params(self,
auth_type_use,
start_date,
end_date,
request_params=None):
"""Build Request Paramaters
Args:
start_date:
end_date:
request_params:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
dict_request_params = {
auth_type_use:
auth_value,
"source":
"multiverse",
"response_timezone":
self.timezone,
"timestamp":
"datehour",
"group": ("advertiser_id,"
"country_id,"
"currency_code,"
"is_reengagement,"
"platform,"
"publisher_id,"
"publisher_sub_ad_id,"
"publisher_sub_adgroup_id,"
"publisher_sub_campaign_id,"
"publisher_sub_publisher_id,"
"publisher_sub_site_id,"
"purchase_validation_status,"
"site_id"),
"fields": ("ad_clicks,"
"ad_clicks_unique,"
"ad_impressions,"
"ad_impressions_unique,"
"ad_network_id,"
"advertiser_id,"
"conversions,"
"country.code,"
"country.name,"
"currency_code,"
"date_hour,"
"events,"
"installs,"
"is_reengagement,"
"payouts,"
"publisher.name,"
"publisher_id,"
"publisher_sub_ad.ref,"
"publisher_sub_adgroup.ref,"
"publisher_sub_campaign.ref,"
"publisher_sub_publisher.ref,"
"publisher_sub_site.ref,"
"site.mobile_app_type,"
"site.package_name,"
"site.store_app_id,"
"site_id"),
"filter":
"({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA),
"start_date":
start_date,
"end_date":
end_date,
"debug":
0
}
if request_params:
self.logger.debug(
"TuneV2AdvertiserStatsActuals: Request",
extra={'request_params': safe_dict(request_params)})
if "fields" in request_params and \
request_params["fields"]:
dict_request_params["fields"] = \
request_params["fields"]
if "group" in request_params and \
request_params["group"]:
dict_request_params["group"] = \
request_params["group"]
if "timestamp" in request_params and \
request_params["timestamp"]:
dict_request_params["timestamp"] = \
request_params["timestamp"]
if "filter" in request_params and \
request_params["filter"]:
dict_request_params["filter"] = "({} AND {})".format(
request_params["filter"],
self._FILTER_NOT_DEBUG_NOR_TEST_DATA)
if "format" in request_params:
dict_request_params["format"] = \
request_params["format"]
if "offset" in request_params:
dict_request_params["offset"] = \
int(request_params["offset"])
if "page" in request_params:
dict_request_params["page"] = \
int(request_params["page"])
if "limit" in request_params:
dict_request_params["limit"] = \
int(request_params["limit"])
if "debug" in request_params:
dict_request_params["debug"] = \
int(request_params["debug"])
response_timezone = None
if "timezone" in request_params:
response_timezone = request_params["timezone"]
if "response_timezone" in request_params:
response_timezone = request_params["response_timezone"]
if response_timezone:
if not validate_tz_name(response_timezone):
return TuneReportingError(
error_message="Invalid Timezone: {}".format(
response_timezone))
self.timezone = response_timezone
dict_request_params["response_timezone"] = \
self.timezone
self.logger.debug(("TuneV2AdvertiserStatsActuals: "
"Timezone: {}").format(self.timezone))
return dict_request_params
def tmc_auth(self, tmc_api_key):
"""TMC Authentication"""
if not tmc_api_key:
raise ValueError("Parameter 'tmc_api_key' not defined.")
self.logger.info("TMC Authentication: Start")
# Attempt to authenticate.
request_url = ('https://api.mobileapptracking.com/v2/advertiser/find')
request_params = {'api_key': tmc_api_key}
auth_request_curl = command_line_request_curl_get(request_url=request_url, request_params=request_params)
try:
auth_response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_label="TMC Authentication"
)
except TuneRequestBaseError as tmc_ex:
self.logger.error("TMC Authentication: Failed", extra=tmc_ex.to_dict())
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC Authentication: Failed: {}".format(get_exception_message(ex)))
raise
if auth_response.status_code != requests.codes.ok:
raise TuneReportingError(
error_message="Invalid request",
error_request_curl=auth_request_curl,
error_code=auth_response.status_code
)
try:
decoded_resp = auth_response.json()
except Exception as ex:
# No JSON response available.
raise TuneReportingError(
error_message='Invalid JSON response: {}'.format(auth_response.text),
errors=ex,
error_request_curl=auth_request_curl,
error_code=TuneReportingErrorCodes.REP_ERR_AUTH_JSON_ERROR
)
tmc_status_code = decoded_resp.get('status_code', None)
tmc_errors = decoded_resp.get('errors', None)
if tmc_errors:
error_code = tmc_status_code
errors = []
if isinstance(tmc_errors, list):
error_list = tmc_errors
error_list = error_list if error_list else []
for error in error_list:
error_message = error.get('message', None)
if error_message:
errors.append(error_message)
if error_message.startswith("Invalid api key"):
error_code = TuneReportingErrorCodes.UNAUTHORIZED
elif isinstance(tmc_errors, dict):
error_message = tmc_errors.get('message', None)
if error_message:
errors.append(error_message)
raise TuneReportingError(
error_message="Error status: {}".format(tmc_errors),
error_request_curl=auth_request_curl,
error_code=error_code,
)
return decoded_resp
def collect(self,
auth_value,
auth_type,
auth_type_use,
site_ids=None,
request_params=None,
site_status=TuneV2AdvertiserSiteStatus.ACTIVE):
"""Collect TMC v2 Advertiser Sites
Args:
tmc_api_key:
site_ids:
request_params:
request_retry:
site_status:
Returns:
"""
if not auth_value:
raise ValueError(
"TuneV2AdvertiserSites: Collect: Value 'auth_value' not provided."
)
if not auth_type:
raise ValueError(
"TuneV2AdvertiserSites: Collect: Value 'auth_type' not valid.")
if not auth_type_use or \
not TuneV2AuthenticationTypes.validate(auth_type_use):
raise ValueError(
"TMC v3 Logs Advertisers Base: Collect: Value 'auth_type_use' not valid."
)
_request_params = {"source": "multiverse"}
if auth_type == TuneV2AuthenticationTypes.API_KEY:
self.api_key = auth_value
if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = tmc_auth_v2_session_token(
tmc_api_key=self.api_key,
logger_level=self.logger_level,
logger_format=self.logger_format)
_request_params.update({
"session_token": self.session_token,
})
else:
_request_params.update({
"api_key": self.api_key,
})
elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = auth_value
_request_params.update({
"session_token": self.session_token,
})
else:
raise ValueError("Invalid 'auth_type': '{}'".format(auth_type))
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller="advertiser/sites",
action="find"
)
self.logger.info("Start Advertiser Sites find")
filter_status = None
if site_status == TuneV2AdvertiserSiteStatus.ACTIVE:
filter_status = "(status = \"active\")"
elif site_status == TuneV2AdvertiserSiteStatus.NOT_ACTIVE:
filter_status = "(status != \"active\")"
filter_sites = None
if site_ids and len(site_ids) > 0:
filter_sites = "(id IN (" + ",".join(site_ids) + "))"
if filter_status and filter_sites:
_request_params["filter"] = "{} AND {}".format(
filter_status, filter_sites)
elif filter_status:
_request_params["filter"] = filter_status
elif filter_sites:
_request_params["filter"] = filter_sites
if request_params:
if "filter" in request_params and request_params["filter"]:
if "filter" in _request_params and _request_params["filter"]:
_request_params["filter"] = "({} AND {})".format(
_request_params["filter"], request_params["filter"])
else:
_request_params["filter"] = "({})".format(
request_params["filter"])
_request_params.update({'limit': request_params.get('limit', 0)})
else:
_request_params.update({'limit': 0})
response = None
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=_request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v2_request_retry_func,
request_retry_excps_func=None,
request_label="{}.{}".format(self.__class__.__name__,
sys._getframe().f_code.co_name))
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TuneV2AdvertiserSites: Collect: Failed",
extra=tmc_req_ex.to_dict(),
)
yield (None, tmc_req_ex)
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TuneV2AdvertiserSites: Collect: Failed",
extra=tmc_rep_ex.to_dict(),
)
yield (None, tmc_rep_ex)
except Exception as ex:
print_traceback(ex)
self.logger.error(get_exception_message(ex))
yield (None, ex)
if response:
json_response = response.json()
if not json_response or \
json_response['status_code'] != 200 or \
'errors' in json_response:
raise TuneReportingError(
error_message="Failed to get advertiser sites: {}, {}".
format(
json_response['status_code'],
json_response['errors'].get('message', None),
),
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
if ('data' not in json_response or not json_response['data']):
raise TuneReportingError(
error_message="Missing 'data': {}".format(
str(json_response)),
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
data = json_response['data']
else:
data = None
if not data or len(data) == 0:
yield ([], None)
else:
for (i, item) in enumerate(data):
yield (item, None)
def tune_v2_request_retry_func(self, response):
"""Request Retry Function
Args:
response:
Returns:
Boolean
"""
try:
response_json = response.json()
except Exception as ex:
# No JSON response available.
raise TuneReportingError(
error_message='Invalid JSON response: {}'.format(response.text),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_JSON_DECODING
)
self.logger.debug("TMC API V2: Check for Retry: Start", extra=response_json)
tune_v2_status_code = None
tune_v2_errors_messages = ""
if 'status_code' in response_json:
tune_v2_status_code = response_json['status_code']
if 'errors' in response_json:
tune_v2_errors = response_json['errors']
if isinstance(tune_v2_errors, dict) \
and 'message' in tune_v2_errors:
tune_v2_errors_messages += tune_v2_errors['message']
elif isinstance(tune_v2_errors, list):
for tune_v2_error in tune_v2_errors:
if isinstance(tune_v2_error, dict) \
and 'message' in tune_v2_error:
tune_v2_errors_messages += tune_v2_error['message']
tune_v2_status_type = get_http_status_type(tune_v2_status_code)
response_extra = {
'status_code': tune_v2_status_code,
'status_type': tune_v2_status_type,
}
if tune_v2_errors_messages:
response_extra.update({'error_messages': safe_str(tune_v2_errors_messages)})
self.logger.debug("TMC API Base: Check for Retry: Response", extra=response_extra)
if tune_v2_status_code == 200:
self.logger.debug("TMC API Base: Check for Retry: Success", extra=response_extra)
return False
if tune_v2_status_code in [401, 403]:
self.logger.error("TMC API: Request: Error", extra={'status_code': tune_v2_status_code})
raise TuneRequestError(
error_message=tune_v2_errors_messages,
error_code=tune_v2_status_code,
)
if tune_v2_status_code in [404, 500]:
if "Api key was not found." in tune_v2_errors_messages:
self.logger.error("TMC API: Request: Error", extra={'tune_v2_status_code': tune_v2_status_code})
raise TuneRequestError(
error_message=tune_v2_errors_messages,
error_code=tune_v2_status_code,
)
self.logger.warning("TMC API Base: Check for Retry: Retry Candidate", extra=response_extra)
return True
self.logger.warning("TMC API Base: Check for Retry: No Retry", extra=response_extra)
return False
def get_advertiser_id(self, auth_type, auth_value, request_retry=None):
"""Get Advertiser ID
Args:
auth_type:
auth_value:
request_retry:
Returns:
"""
if not auth_type:
raise ValueError(
"TMC v2 Advertisers: Get Advertiser ID: Value 'auth_type' not provided."
)
if not auth_value:
raise ValueError(
"TMC v2 Advertisers: Get Advertiser ID: Value 'auth_value' not provided."
)
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller="advertiser",
action="find"
)
request_params = {auth_type: auth_value}
self.logger.info("TMC v2 Advertisers: Advertiser ID")
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v2_request_retry_func,
request_retry_excps_func=None,
request_label="TMC v2 Advertisers")
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertisers: Advertiser ID: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertisers: Advertiser ID: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v2 Advertisers: Advertiser ID: Failed",
extra={'error': get_exception_message(ex)},
)
raise TuneReportingError(
error_message=("TMC v2 Advertisers: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Advertisers: Advertiser ID:")
self.logger.debug("TMC v2 Advertisers: Advertiser ID",
extra={'response': json_response})
json_response_status_code = json_response['status_code']
http_status_successful = is_http_status_type(
http_status_code=json_response_status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
if not http_status_successful or not json_response['data']:
raise TuneReportingError(
error_message="TMC v2 Advertisers: Failed: {}".format(
json_response_status_code),
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
if 'data' not in json_response or \
not json_response['data'] or \
len(json_response['data']) == 0:
raise TuneReportingError(
error_message="TMC v2 Advertisers: Advertiser ID: Failed",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
advertiser_data = json_response['data'][0]
self.advertiser_id = advertiser_data.get('id', None)
if self.advertiser_id is None:
raise TuneReportingError(
error_message="TMC v2 Advertisers: Advertiser ID: Failed",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
self.logger.info("TMC v2 Advertisers: Advertiser ID: {}".format(
self.advertiser_id))
return True
def get_session_token(self, tmc_api_key, request_retry=None):
"""Generate session token is returned to provide
access to service.
Args:
tmc_api_key:
request_retry:
Returns:
"""
self.logger.info("TMC v2 Session Authenticate: Get Token")
self.api_key = tmc_api_key
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller="session/authenticate",
action="api_key"
)
request_params = \
{
'api_keys': tmc_api_key,
"source": "multiverse"
}
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v2_request_retry_func,
request_retry_excps_func=None,
request_label="TMC v2 Session Authenticate")
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Session Authenticate: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Session Authenticate: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v2 Session Authenticate: Failed: {}".format(
get_exception_message(ex)))
raise TuneReportingError(
error_message=(
"TMC v2 Session Authenticate: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Get Session Token: Validate",
)
self.logger.debug(
"TMC v2 Session Authenticate: Details:",
extra={'response': json_response},
)
response_status_code = json_response.get('status_code', None)
status_code_successful = is_http_status_type(
http_status_code=response_status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
response_errors = json_response.get('errors', None)
response_error_message = ""
if response_errors:
if isinstance(response_errors, dict):
error_message = response_errors.get('message', None)
if error_message:
if error_message.startswith("Invalid api key"):
response_status_code = TuneReportingErrorCodes.UNAUTHORIZED
response_error_message += error_message
elif isinstance(response_errors, list):
for response_error in response_errors:
if isinstance(response_error, dict) \
and 'message' in response_error:
error_message = response_error.get('message', None)
if error_message:
if error_message.startswith("Invalid api key"):
response_status_code = TuneReportingErrorCodes.UNAUTHORIZED
response_error_message += error_message
error_code = response_status_code
if not status_code_successful or not json_response['data']:
raise TuneReportingError(
error_message="TMC v2 Session Authenticate: Failed: {}".format(
response_error_message),
error_code=error_code,
)
self.session_token = json_response['data']
self.logger.info("TMC v2 Session Authenticate",
extra={'session_token': self.session_token})
self.logger.info("TMC v2 Session Authenticate: Finished")
return True
def _collect(self,
auth_type_use,
start_date,
end_date,
request_params=None,
request_retry=None,
request_action=TuneV2AdvertiserStatsActions.FIND):
"""Collect data: TUNE Advertiser Stats Actuals.
Args:
start_date:
end_date:
request_params:
request_retry:
request_action:
Returns:
"""
self.logger.debug(("TuneV2AdvertiserStatsActuals: Collect: "
"Action: '{}'").format(request_action))
dict_request_params = self._map_request_params(auth_type_use,
start_date, end_date,
request_params)
self.logger.debug("TuneV2AdvertiserStatsActuals: Collect",
extra={'build_params': dict_request_params})
try:
if request_action == TuneV2AdvertiserStatsActions.FIND:
self._find_v2(request_params=dict_request_params,
request_retry=request_retry)
elif request_action == TuneV2AdvertiserStatsActions.EXPORT:
self._export_v2_download_csv(
auth_type_use=auth_type_use,
export_controller=self._CONTROLLER,
export_action='export',
export_status_controller=self._CONTROLLER,
export_status_action='status',
request_params=dict_request_params,
request_retry=request_retry)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Collect: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Collect: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TuneV2AdvertiserStatsActuals: Collect: {}".format(
get_exception_message(ex)))
raise TuneReportingError(
error_message=(
"TuneV2AdvertiserStatsActuals: Collect: Failed: {}"
).format(get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
def _export_v3_job_to_queue(self, request_params, request_retry=None, request_label="TMC v3 Job On Queue"):
"""Gather data using action export.json.
:param request_params:
:param request_retry:
:return:
"""
request_url = \
self.tune_mat_request_path(
mat_api_version="v3",
controller=self.controller,
action="exports/{}".format(
self.logs_advertisers_type
)
)
self.logger.debug(
msg=("TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'exports': "
"Start Advertiser Report Actuals").format(self.logs_advertisers_type),
extra={'start_date': request_params["start_date"],
'end_date': request_params["end_date"]}
)
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=request_retry,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v3_request_retry_func,
request_retry_excps_func=None,
request_label=request_label
)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v3 Logs Advertisers Base: {}".format(get_exception_message(ex)))
raise
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v3 Logs Advertisers: '{}': Action 'exports'".format(self.logs_advertisers_type)
)
if ('handle' not in json_response or not json_response['handle']):
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'exports': "
"Response missing 'handle': {}"
).format(self.logs_advertisers_type, str(json_response)),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE
)
export_job = json_response['handle']
if not export_job:
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'exports': "
"Response missing 'handle'"
).format(self.logs_advertisers_type),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE
)
self.logger.info(
"TuneV3LogsAdvertisersBase",
extra={
'advertiser_type': self.logs_advertisers_type,
'action': 'exports',
'start_date': request_params["start_date"],
'end_date': request_params["end_date"],
'job': export_job
}
)
return export_job
def stream(
self,
auth_value,
auth_type,
auth_type_use,
start_date,
end_date,
request_params,
request_retry=None,
):
"""Stream data: TUNE Advertiser Stats Base
Args:
auth_value:
auth_type:
start_date:
end_date:
request_params:
request_retry:
Returns:
"""
self.logger.info("TMC v2 Advertiser Stats: Stream")
if not auth_value:
raise ValueError(
error_message=
"TMC v2 Advertiser Stats: Stream: Value 'auth_value' not provided."
)
if not auth_type or \
not TuneV2AuthenticationTypes.validate(auth_type):
raise ValueError(
error_message=
"TMC v2 Advertiser Stats: Stream: Value 'auth_type' not valid."
)
if not auth_type_use or \
not TuneV2AuthenticationTypes.validate(auth_type_use):
raise ValueError(
error_message=
"TMC v2 Advertiser Stats: Stream: Value 'auth_type_use' not valid."
)
if not request_params:
raise ValueError(
error_message="Value 'request_params' not provided.")
if auth_type == TuneV2AuthenticationTypes.API_KEY:
self.api_key = auth_value
if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = tmc_auth_v2_session_token(
tmc_api_key=self.api_key,
logger_level=self.logger_level,
logger_format=self.logger_format)
elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = auth_value
else:
raise ValueError(
error_message="Invalid 'auth_type': '{}'".format(auth_type))
auth_value_use = None
if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
if self.session_token is None:
raise TuneReportingError(
error_message="Value 'session_token' not defined.",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
auth_value_use = self.session_token
elif auth_type_use == TuneV2AuthenticationTypes.API_KEY:
if self.api_key is None:
raise TuneReportingError(
error_message="Value 'api_key' not defined.",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE,
)
auth_value_use = self.api_key
if not auth_value_use:
raise TuneReportingError(
error_message="Value 'auth_value_use' not defined.",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
return self._stream_v2(auth_type_use=auth_type_use,
start_date=start_date,
end_date=end_date,
request_params=request_params,
request_retry=request_retry)
def _check_v3_job_status_on_queue(self, export_job, request_retry=None, request_label="TMC v3 Job Status On Queue"):
"""Status of Export Report.
Args:
export_job:
request_retry:
Returns:
"""
request_label = "v3 Logs Advertisers Check Export Status"
request_url = \
self.tune_mat_request_path(
mat_api_version="v3",
controller=self.controller,
action="exports/{}".format(
export_job
)
)
self.logger.info((
"TMC v3 Logs Advertisers Base: Check Export Status: "
"Logs '{}': "
"Action: 'exports status', "
"Status of Export Report for "
"Job Handle: '{}'"
).format(self.logs_advertisers_type, export_job))
tries = -1 # default: -1 (indefinite)
delay = 10
jitter = 0
max_delay = 60
if request_retry:
if 'delay' in request_retry:
delay = request_retry['delay']
if 'jitter' in request_retry:
jitter = request_retry['jitter']
if 'max_delay' in request_retry:
max_delay = request_retry['max_delay']
request_params = {"session_token": self.session_token}
self.logger.debug("TMC v3 Logs Advertisers Base: Check Export Status", extra={'request_url': request_url})
self.logger.debug(
"TMC v3 Logs Advertisers Base: Check Export Status: Request Retry",
extra={'tries': tries,
'delay': delay,
'jitter': jitter,
'max_delay': max_delay}
)
self.logger.debug(
"TMC v3 Logs Advertisers Base: Check Export Status: Request",
extra={'request_params': safe_dict(request_params)}
)
report_url = None
_attempts = 1
export_percent_complete = 0
export_status_action = 'exports status'
self.logger.warning(
"TMC v3 Logs Advertisers Base: Check Export Status",
extra={'job': export_job,
'attempt': _attempts,
'action': export_status_action}
)
time.sleep(10)
_tries, _delay = tries, delay
while True:
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v3_request_retry_func,
request_retry_excps_func=None,
request_label=request_label
)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra={'error': get_exception_message(ex)}
)
raise
http_status_successful = is_http_status_type(
http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
if not http_status_successful:
raise TuneReportingError(
error_message="Failed to get export status on queue: {}".format(response.status_code),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST
)
json_response = response.json()
export_percent_complete = 0
if "percent_complete" in json_response:
export_percent_complete = \
safe_int(json_response["percent_complete"])
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status",
extra={
'job': export_job,
'response_status_code': json_response["status"],
'export_percent_complete': safe_int(export_percent_complete)
}
)
if (export_percent_complete == 100 and json_response["status"] == "complete" and json_response["url"]):
report_url = json_response["url"]
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status: Completed",
extra={
'job': export_job,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': safe_int(export_percent_complete)
}
)
break
if tries >= 0:
_tries -= 1
if _tries == 0:
self.logger.error(("TMC v3 Logs Advertisers Base: "
"Check Job Export Status: Exhausted Retries"),
extra={
'attempt': _attempts,
'tries': _tries,
'action': export_status_action,
'request_label': request_label,
'export_percent_complete': export_percent_complete
})
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Check Job Export Status: "
"Exhausted Retries: "
"Percent Completed: {}"
).format(export_percent_complete),
error_code=TuneReportingErrorCodes.REP_ERR_JOB_STOPPED
)
_attempts += 1
self.logger.warning(
"TMC v3 Logs Advertisers Base: Check Export Status",
extra={'attempt': _attempts,
'job': export_job,
'delay': _delay,
'action': 'exports status'}
)
time.sleep(_delay)
_delay += jitter
_delay = min(_delay, max_delay)
if export_percent_complete == 100 and not report_url:
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: Check Job Export Status: "
"Download report URL: Undefined"
)
)
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status: Finished",
extra={
'attempt': _attempts,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': export_percent_complete,
'job': export_job
}
)
return report_url
def _find_v2(
self,
request_params,
request_retry=None,
request_label="TMC v2 Advertiser Stats Find",
):
"""Gather data using action find.json.
"""
self.logger.debug("TuneV2AdvertiserStatsBase",
extra={'action': 'find'})
if "start_date" not in request_params:
raise ValueError(
"Missing attribute 'start_date' in parameter 'request_params'")
request_start_date = request_params["start_date"]
if "end_date" not in request_params:
raise ValueError(
"Missing attribute 'end_date' in parameter 'request_params'")
request_end_date = request_params["end_date"]
datetime_start = dt.datetime.strptime(request_start_date, '%Y-%m-%d')
datetime_end = dt.datetime.strptime(request_end_date, '%Y-%m-%d')
str_date_start = str(datetime_start.date())
str_date_end = str(datetime_end.date())
self.logger.debug(
("TuneV2AdvertiserStatsBase"),
extra={
'action': 'find',
'start_date': str_date_start,
'end_date': str_date_end
})
if not request_retry:
request_retry = {'timeout': 60}
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=self.controller,
action="find"
)
if 'format' not in request_params:
request_params['format'] = TuneV2AdvertiserStatsFormats.JSON
request_params["start_date"] += " 00:00:00"
request_params["end_date"] += " 23:59:59"
try:
response = self.mv_request.request(request_method="GET",
request_url=request_url,
request_params=request_params,
request_label=request_label)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v2 Advertiser Stats: {}".format(
get_exception_message(ex)))
raise
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Advertiser Stats: Action 'find'")
if json_response['status_code'] != 200:
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Action 'find': "
"Failed to find stats: {}, {}").format(
json_response['status_code'],
json.dumps(json_response),
),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
data = json_response['data']
self.data = data
def _collect(
self,
auth_type_use,
start_date,
end_date,
request_params=None,
request_retry=None,
request_action=TuneV3LogsAdvertisersActions.FIND
):
"""Collect data: TUNE Advertiser Stats Clicks.
Args:
start_date:
end_date:
request_params:
request_retry:
request_action:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
dict_request_params = {
auth_type_use: auth_value,
"timezone": self.timezone,
"fields": (
"created,"
"ad_network_id,"
"campaign.id,"
"campaign.name,"
"publisher.id,"
"publisher.name,"
"publisher_ref_id,"
"publisher_sub_site.id,"
"publisher_sub_site.ref,"
"publisher_sub_site.name,"
"publisher_sub_campaign.id,"
"publisher_sub_campaign.ref,"
"publisher_sub_campaign.name,"
"request_url,"
"site.id,"
"site.mobile_app_type,"
"site.name,"
"site.package_name,"
"site.store_app_id"
),
"filter": "({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA),
"start_date": start_date,
"end_date": end_date,
"debug": 0
}
if request_params:
if "fields" in request_params and request_params["fields"]:
dict_request_params["fields"] = \
request_params["fields"]
if "group" in request_params and request_params["group"]:
dict_request_params["group"] = \
request_params["group"]
if "timestamp" in request_params and request_params["timestamp"]:
dict_request_params["timestamp"] = \
request_params["timestamp"]
if "filter" in request_params and request_params["filter"]:
dict_request_params["filter"] = "({} AND {})".format(
request_params["filter"], self._FILTER_NOT_DEBUG_NOR_TEST_DATA
)
if "limit" in request_params:
dict_request_params["limit"] = \
int(request_params["limit"])
if "debug" in request_params:
dict_request_params["debug"] = \
int(request_params["debug"])
timezone = None
if "timezone" in request_params:
timezone = request_params["timezone"]
if timezone:
if not validate_tz_name(timezone):
return TuneReportingError(error_message="Invalid Timezone: {}".format(timezone))
self.timezone = timezone
dict_request_params["timezone"] = \
self.timezone
self.logger.debug(("TMC v3 Logs Advertisers Clicks: "
"Action '{}', Params: {}").format(request_action, str(dict_request_params)))
self.logger.debug(("TMC v3 Logs Advertisers Clicks: " "Timezone: {}").format(self.timezone))
try:
if request_action == TuneV3LogsAdvertisersActions.FIND:
request_params["sorts"] = "created desc"
self._find_v3(request_params=dict_request_params, request_retry=request_retry)
elif request_action == TuneV3LogsAdvertisersActions.EXPORT:
self._export_v3_download_csv(request_params=dict_request_params, request_retry=request_retry)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Clicks: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Clicks: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v3 Logs Advertisers Clicks: Failed: {}".format(get_exception_message(ex)))
raise TuneReportingError(
error_message=("TMC v3 Logs Advertisers Clicks: Failed: {}").format(get_exception_message(ex)),
errors=ex
)
def _stream_v2(
self,
auth_type_use,
start_date,
end_date,
request_params,
request_retry=None,
):
"""Stream data: TUNE Advertiser Stats Actuals.
Args:
start_date:
end_date:
request_params:
request_retry:
Returns:
"""
self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export",
extra={'request_params': request_params})
dict_request_params = self._map_request_params(auth_type_use,
start_date, end_date,
request_params)
self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export",
extra={'build_params': dict_request_params})
try:
response = self._export_stream_v2(
auth_type_use,
export_controller=self._CONTROLLER,
export_action='export',
export_status_controller=self._CONTROLLER,
export_status_action='status',
request_params=dict_request_params,
request_retry=request_retry)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Stream: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Stream: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TuneV2AdvertiserStatsActuals: {}".format(
get_exception_message(ex)))
raise TuneReportingError(
error_message=(
"TuneV2AdvertiserStatsActuals: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
return response