def main(tmc_api_key):
tune_v3_logs_advertisers_impressions_export = \
TuneV3LogsAdvertisersImpressions(
logger_level=logging.INFO,
logger_format=LoggingFormat.JSON,
logger_output=LoggingOutput.STDOUT_COLOR
)
tune_v3_logs_advertisers_impressions_export.logger_level = logging.DEBUG
tz = pytz.timezone("America/New_York")
yesterday = datetime.now(tz).date() - timedelta(days=1)
str_yesterday = str(yesterday)
try:
tune_v3_logs_advertisers_impressions_export.collect(
auth_value=tmc_api_key,
auth_type=TuneV2AuthenticationTypes.API_KEY,
auth_type_use=TuneV2AuthenticationTypes.SESSION_TOKEN,
start_date=str_yesterday,
end_date=str_yesterday,
advertiser_id=877,
request_params={'timezone': 'America/New_York',
'filter': "(ad_network_id <> 0)",
'limit': 5},
request_action=TuneV3LogsAdvertisersActions.EXPORT
)
except TuneRequestBaseError as tmc_req_ex:
print_traceback(tmc_req_ex)
pprint(tmc_req_ex.to_dict())
print(str(tmc_req_ex))
except TuneReportingError as tmc_rep_ex:
pprint(tmc_rep_ex.to_dict())
print(str(tmc_rep_ex))
except Exception as ex:
print_traceback(ex)
print(get_exception_message(ex))
for row in list(tune_v3_logs_advertisers_impressions_export.generator):
pprint(row)
def tmc_auth_v2_session_token(
tmc_api_key,
logger_level=logging.NOTSET,
logger_format=TuneLoggingFormat.JSON,
):
"""TMC Authentication
:return:
"""
log.info("TMC v2 Session Token: Request")
try:
tune_v2_session_authenticate = \
TuneV2SessionAuthenticate(
logger_level=logger_level,
logger_format=logger_format
)
if tune_v2_session_authenticate.get_session_token(tmc_api_key=tmc_api_key, request_retry=None):
session_token = tune_v2_session_authenticate.session_token
except TuneReportingError as auth_ex:
log.error("TMC v2 Session Token: Failed", extra=auth_ex.to_dict())
raise
except Exception as ex:
print_traceback(ex)
log.error(
'TMC v2 Session Token: Failed: Unexpected',
extra={'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)}
)
raise TuneReportingAuthError(
error_message="TMC v2 Session Token: Failed: Unexpected",
errors=ex,
)
log.debug("TMC v2 Session Token: Response", extra={'session_token': session_token})
return session_token
def _find_v3(self, request_params, request_retry=None, request_label="TMC v3 Logs Advertisers"):
"""Gather data using action 'find'
Args:
request_params:
request_retry:
Returns:
"""
self.logger.debug(("TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'find'").format(self.logs_advertisers_type))
if "start_date" not in request_params:
raise ValueError("Missing attribute 'start_date' in parameter 'request_params'")
request_start_date = request_params["start_date"]
if "end_date" not in request_params:
raise ValueError("Missing attribute 'end_date' in parameter 'request_params'")
request_end_date = request_params["end_date"]
datetime_start = dt.datetime.strptime(request_start_date, '%Y-%m-%d')
datetime_end = dt.datetime.strptime(request_end_date, '%Y-%m-%d')
str_date_start = str(datetime_start.date())
str_date_end = str(datetime_end.date())
self.logger.debug(
"TMC v3 Logs Advertisers Base: Action 'find'",
extra={'start_date': str_date_start,
'end_date': str_date_end}
)
if not request_retry:
request_retry = {'timeout': 600}
request_url = \
self.tune_mat_request_path(
mat_api_version="v3",
controller=self.controller,
action=self.logs_advertisers_type
)
if 'response_format' not in request_params:
request_params['response_format'] = \
TuneV3LogsAdvertisersResponseFormats.JSON
request_params["start_date"] += "T00:00:00Z"
request_params["end_date"] += "T23:59:59Z"
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_label=request_label
)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v3 Logs Advertisers Base: {}".format(get_exception_message(ex)))
raise
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v3 Logs Advertisers: Action 'find'",
)
data = json_response['data']
self.data = data
def main(tmc_api_key):
TIMEZONE_COLLECT = "America/New_York"
export_format = TuneV2AdvertiserStatsFormats.CSV
tune_v2_advertiser_stats_actuals = \
TuneV2AdvertiserStatsActuals(
timezone=TIMEZONE_COLLECT,
logger_level=logging.INFO,
logger_format=LoggingFormat.JSON,
logger_output=LoggingOutput.STDOUT_COLOR
)
dw_file_path = "data.{}".format(export_format)
if os.path.exists(dw_file_path):
os.remove(dw_file_path)
tz = pytz.timezone(TIMEZONE_COLLECT)
yesterday = datetime.now(tz).date() - timedelta(days=1)
str_yesterday = str(yesterday)
try:
tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key)
response = tune_v2_advertiser_stats_actuals.stream(
auth_value=tmc_api_key,
auth_type=TuneV2AuthenticationTypes.API_KEY,
auth_type_use=TuneV2AuthenticationTypes.API_KEY,
start_date=str_yesterday,
end_date=str_yesterday,
request_params={
'format': export_format,
'fields': (
"ad_clicks,"
"ad_clicks_unique,"
"ad_impressions,"
"ad_impressions_unique,"
"ad_network_id,"
"advertiser_id,"
"country.code,"
"date_hour,"
"events,"
"installs,"
"is_reengagement,"
"payouts,"
"publisher_id,"
"publisher_sub_ad.ref,"
"publisher_sub_adgroup.ref,"
"publisher_sub_campaign.ref,"
"publisher_sub_publisher.ref,"
"publisher_sub_site.ref,"
"site_id"
),
'group': (
"country_id,"
"is_reengagement,"
"publisher_id,"
"publisher_sub_ad_id,"
"publisher_sub_adgroup_id,"
"publisher_sub_campaign_id,"
"publisher_sub_publisher_id,"
"publisher_sub_site_id,"
"site_id"
),
'timezone': "America/Los_Angeles"
},
request_retry={'delay': 15,
'timeout': 30,
'tries': 10}
)
with open(file=dw_file_path, mode='wb') as dw_file_wb:
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
dw_file_wb.write(chunk)
dw_file_wb.flush()
with open(file=dw_file_path, mode='r') as csv_file_r:
csv_dict_reader = csv.DictReader(csv_file_r)
for row in csv_dict_reader:
print(row)
statinfo = os.stat(dw_file_path)
extra = {
'response_status_code': response.status_code,
'response_headers': response.headers,
'dw_file_path': dw_file_path,
'dw_file_size': convert_size(statinfo.st_size)
}
pprint(extra)
except TuneRequestBaseError as tmc_req_ex:
print_traceback(tmc_req_ex)
pprint(tmc_req_ex.to_dict())
print(str(tmc_req_ex))
except TuneReportingError as tmc_rep_ex:
pprint(tmc_rep_ex.to_dict())
print(str(tmc_rep_ex))
except Exception as ex:
print_traceback(ex)
print(get_exception_message(ex))
def _check_v2_job_status_on_queue(
self,
auth_type,
auth_value,
export_status_controller,
export_status_action,
export_job_id,
request_retry=None,
):
"""Check Job Export Status
Args:
export_status_controller:
export_status_action:
export_job_id:
request_retry:
Returns:
"""
request_label = "TMC v2 Advertiser Stats: Check Export Status"
v2_export_status_request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=export_status_controller,
action=export_status_action
)
request_params = {auth_type: auth_value, "job_id": export_job_id}
self.logger.info(
("TMC v2 Advertiser Stats: Check Job Status"),
extra={
'action': export_status_action,
'job_id': export_job_id,
'request_url': v2_export_status_request_url,
'request_params': safe_dict(request_params)
})
tries = 60 # -1 (indefinite)
delay = 10
jitter = 10
max_delay = 60
if request_retry is not None:
if 'delay' in request_retry:
delay = request_retry['delay']
if 'jitter' in request_retry:
jitter = request_retry['jitter']
if 'max_delay' in request_retry:
max_delay = request_retry['max_delay']
if 'tries' in request_retry:
tries = request_retry['tries']
else:
request_retry.update({'tries': 60})
else:
request_retry = {'tries': 60, 'delay': 10, 'timeout': 60}
self.logger.debug(msg=("TMC v2 Advertiser Stats: Check Job Status: "
"Request Retry"),
extra=request_retry)
report_url = None
_attempts = 1
export_percent_complete = 0
time.sleep(10)
_tries, _delay = tries, delay
while True:
try:
response = self.mv_request.request(
request_method="GET",
request_url=v2_export_status_request_url,
request_params=request_params,
request_label=request_label,
request_retry_func=self.tune_v2_request_retry_func)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v2 Advertiser Stats: Check Job Status: {}".format(
get_exception_message(ex)))
raise
http_status_successful = is_http_status_type(
http_status_code=response.status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
if not http_status_successful:
raise TuneReportingError(
error_message=(
"Failed to get export status on queue: {}").format(
response.status_code),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
if hasattr(response, 'url'):
self.logger.info(
"TMC v2 Advertiser Stats: Reporting API: Status URL",
extra={'response_url': response.url})
json_response = response.json()
if not json_response:
request_status_successful = False
elif 'status_code' not in json_response:
request_status_successful = False
else:
status_code = json_response['status_code']
request_status_successful = is_http_status_type(
http_status_code=status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
errors = None
if 'errors' in json_response:
errors = json_response['errors']
if not request_status_successful:
error_message = (
"TMC v2 Advertiser Stats: Check Job Status: GET '{}', Failed: {}, {}"
).format(v2_export_status_request_url, status_code, errors)
if (status_code ==
TuneReportingError.EX_SRV_ERR_500_INTERNAL_SERVER):
self.logger.error(error_message)
elif (status_code ==
TuneReportingError.EX_SRV_ERR_503_SERVICE_UNAVAILABLE):
self.logger.error(error_message)
elif (status_code ==
TuneReportingError.EX_SRV_ERR_504_SERVICE_TIMEOUT):
self.logger.error(error_message)
continue
elif (status_code ==
TuneReportingError.EX_CLT_ERR_408_REQUEST_TIMEOUT):
self.logger.error(
"GET '{}' request timeout, Retrying: {}".format(
v2_export_status_request_url, status_code))
continue
else:
raise TuneReportingError(error_message=error_message,
error_code=status_code)
if tries >= 0 and _tries <= 1:
if (status_code == HttpStatusCode.GATEWAY_TIMEOUT):
raise TuneReportingError(
error_message=error_message,
error_code=TuneReportingErrorCodes.GATEWAY_TIMEOUT)
elif (status_code == HttpStatusCode.REQUEST_TIMEOUT):
raise TuneReportingError(
error_message=error_message,
error_code=TuneReportingErrorCodes.REQUEST_TIMEOUT)
else:
raise TuneReportingError(error_message=error_message,
error_code=status_code)
else:
self.logger.warning(error_message)
export_percent_complete = 0
if 'data' in json_response and json_response['data']:
json_data = json_response['data']
if "percent_complete" in json_data:
export_percent_complete = \
safe_int(json_data["percent_complete"])
self.logger.info(msg=("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"Response Success"),
extra={
'job_id':
export_job_id,
'export_status':
json_data["status"],
'export_percent_complete':
safe_int(export_percent_complete),
'attempt':
_attempts
})
if (export_status_action ==
TuneV2AdvertiserStatsStatusAction.STATUS):
if (export_percent_complete == 100
and json_data["status"] == "complete"
and json_data["url"]):
report_url = json_data["url"]
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Completed"),
extra={
'job_id': export_job_id,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label
})
break
elif (export_status_action ==
TuneV2AdvertiserStatsStatusAction.DOWNLOAD):
if (export_percent_complete == 100
and json_data["status"] == "complete"
and json_data["data"]["url"]):
report_url = json_data["data"]["url"]
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Completed"),
extra={
'job_id': export_job_id,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label
})
break
else:
self.logger.debug("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"No Data Available")
if tries >= 0:
_tries -= 1
if _tries == 0:
self.logger.error(
("TMC v2 Advertiser Stats: "
"Check Job Export Status: Exhausted Retries"),
extra={
'attempt':
_attempts,
'tries':
_tries,
'action':
export_status_action,
'request_label':
request_label,
'export_percent_complete':
safe_int(export_percent_complete),
'job_id':
export_job_id
})
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Check Job Export Status: "
"Exhausted Retries: "
"Percent Completed: {}").format(
safe_int(export_percent_complete)),
error_code=TuneReportingErrorCodes.
REP_ERR_RETRY_EXHAUSTED)
_attempts += 1
self.logger.info("TMC v2 Advertiser Stats: Check Job Status",
extra={
'attempt': _attempts,
'job_id': export_job_id,
'delay': _delay,
'action': export_status_action
})
time.sleep(_delay)
_delay += jitter
_delay = min(_delay, max_delay)
if export_percent_complete == 100 and not report_url:
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: Check Job Export Status: "
"Download report URL: Undefined"),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
self.logger.info(
"TMC v2 Advertiser Stats: Check Job Export Status: Finished",
extra={
'attempt': _attempts,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': export_percent_complete,
'job_id': export_job_id
})
return report_url
def _export_v2_job_to_queue(
self,
export_controller,
export_action,
request_params,
request_retry=None,
request_label="TMC v2 Job To Queue",
):
"""Export Report Request to Job Queue
Args:
export_controller:
export_action:
request_params:
request_retry:
Returns:
Export Job ID
"""
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=export_controller,
action=export_action
)
self.logger.debug(
("TMC v2 Advertiser Stats: "
"Start Advertiser Report Actuals"),
extra={
'action': export_action,
'start_date': request_params["start_date"],
'end_date': request_params["end_date"]
})
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_label=request_label,
request_retry_func=self.tune_v2_request_retry_func)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Request Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Reporting Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(get_exception_message(ex))
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: Unexpected Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Advertiser Stats: Action '{}'".format(
export_action))
if hasattr(response, 'url'):
self.logger.info(
"TMC v2 Advertiser Stats: Reporting API: Export URL",
extra={'response_url': response.url})
if (not json_response or json_response['status_code'] != 200
or 'errors' in json_response):
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Action '{}': "
"Failed to export stats: {}, {}").format(
export_action, json_response['status_code'],
json_response['errors']),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
if ('data' not in json_response or not json_response['data']):
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Action '{}': "
"Missing data").format(export_action),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
json_data = json_response['data']
export_job_id = None
if export_action == TuneV2AdvertiserStatsExportActions.EXPORT:
if ('job_id' not in json_data or not json_data['job_id']):
raise TuneReportingError(error_message=(
"TMC v2 Advertiser Stats: "
"Action '{}': "
"Response missing 'export_job_id': {}").format(
export_action, str(json_data)),
error_code=TuneReportingErrorCodes.
REP_ERR_UNEXPECTED_VALUE)
export_job_id = json_data['job_id']
elif export_action == TuneV2AdvertiserStatsExportActions.FIND_EXPORT_QUEUE:
export_job_id = json_data
if not export_job_id:
raise TuneReportingError(
error_message=(
"TMC v2 Advertiser Stats: "
"Action '{}': "
"Response missing 'job_id'").format(export_action),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE)
self.logger.info("TMC v2 Advertiser Stats: Reporting API: Job ID",
extra={'job_id': export_job_id})
return export_job_id
def _find_v2(
self,
request_params,
request_retry=None,
request_label="TMC v2 Advertiser Stats Find",
):
"""Gather data using action find.json.
"""
self.logger.debug("TuneV2AdvertiserStatsBase",
extra={'action': 'find'})
if "start_date" not in request_params:
raise ValueError(
"Missing attribute 'start_date' in parameter 'request_params'")
request_start_date = request_params["start_date"]
if "end_date" not in request_params:
raise ValueError(
"Missing attribute 'end_date' in parameter 'request_params'")
request_end_date = request_params["end_date"]
datetime_start = dt.datetime.strptime(request_start_date, '%Y-%m-%d')
datetime_end = dt.datetime.strptime(request_end_date, '%Y-%m-%d')
str_date_start = str(datetime_start.date())
str_date_end = str(datetime_end.date())
self.logger.debug(
("TuneV2AdvertiserStatsBase"),
extra={
'action': 'find',
'start_date': str_date_start,
'end_date': str_date_end
})
if not request_retry:
request_retry = {'timeout': 60}
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller=self.controller,
action="find"
)
if 'format' not in request_params:
request_params['format'] = TuneV2AdvertiserStatsFormats.JSON
request_params["start_date"] += " 00:00:00"
request_params["end_date"] += " 23:59:59"
try:
response = self.mv_request.request(request_method="GET",
request_url=request_url,
request_params=request_params,
request_label=request_label)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertiser Stats: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v2 Advertiser Stats: {}".format(
get_exception_message(ex)))
raise
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Advertiser Stats: Action 'find'")
if json_response['status_code'] != 200:
raise TuneReportingError(
error_message=("TMC v2 Advertiser Stats: "
"Action 'find': "
"Failed to find stats: {}, {}").format(
json_response['status_code'],
json.dumps(json_response),
),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST)
data = json_response['data']
self.data = data
def _collect(
self,
auth_type_use,
start_date,
end_date,
request_params=None,
request_retry=None,
request_action=TuneV3LogsAdvertisersActions.FIND
):
"""Collect data: TUNE Advertiser Stats Clicks.
Args:
start_date:
end_date:
request_params:
request_retry:
request_action:
Returns:
"""
auth_value = None
if auth_type_use == TuneV2AuthenticationTypes.API_KEY:
auth_value = self.api_key
elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
auth_value = self.session_token
dict_request_params = {
auth_type_use: auth_value,
"timezone": self.timezone,
"fields": (
"created,"
"ad_network_id,"
"campaign.id,"
"campaign.name,"
"publisher.id,"
"publisher.name,"
"publisher_ref_id,"
"publisher_sub_site.id,"
"publisher_sub_site.ref,"
"publisher_sub_site.name,"
"publisher_sub_campaign.id,"
"publisher_sub_campaign.ref,"
"publisher_sub_campaign.name,"
"request_url,"
"site.id,"
"site.mobile_app_type,"
"site.name,"
"site.package_name,"
"site.store_app_id"
),
"filter": "({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA),
"start_date": start_date,
"end_date": end_date,
"debug": 0
}
if request_params:
if "fields" in request_params and request_params["fields"]:
dict_request_params["fields"] = \
request_params["fields"]
if "group" in request_params and request_params["group"]:
dict_request_params["group"] = \
request_params["group"]
if "timestamp" in request_params and request_params["timestamp"]:
dict_request_params["timestamp"] = \
request_params["timestamp"]
if "filter" in request_params and request_params["filter"]:
dict_request_params["filter"] = "({} AND {})".format(
request_params["filter"], self._FILTER_NOT_DEBUG_NOR_TEST_DATA
)
if "limit" in request_params:
dict_request_params["limit"] = \
int(request_params["limit"])
if "debug" in request_params:
dict_request_params["debug"] = \
int(request_params["debug"])
timezone = None
if "timezone" in request_params:
timezone = request_params["timezone"]
if timezone:
if not validate_tz_name(timezone):
return TuneReportingError(error_message="Invalid Timezone: {}".format(timezone))
self.timezone = timezone
dict_request_params["timezone"] = \
self.timezone
self.logger.debug(("TMC v3 Logs Advertisers Clicks: "
"Action '{}', Params: {}").format(request_action, str(dict_request_params)))
self.logger.debug(("TMC v3 Logs Advertisers Clicks: " "Timezone: {}").format(self.timezone))
try:
if request_action == TuneV3LogsAdvertisersActions.FIND:
request_params["sorts"] = "created desc"
self._find_v3(request_params=dict_request_params, request_retry=request_retry)
elif request_action == TuneV3LogsAdvertisersActions.EXPORT:
self._export_v3_download_csv(request_params=dict_request_params, request_retry=request_retry)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Clicks: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Clicks: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v3 Logs Advertisers Clicks: Failed: {}".format(get_exception_message(ex)))
raise TuneReportingError(
error_message=("TMC v3 Logs Advertisers Clicks: Failed: {}").format(get_exception_message(ex)),
errors=ex
)
def collect(self,
auth_value,
auth_type,
auth_type_use,
site_ids=None,
request_params=None,
site_status=TuneV2AdvertiserSiteStatus.ACTIVE):
"""Collect TMC v2 Advertiser Sites
Args:
tmc_api_key:
site_ids:
request_params:
request_retry:
site_status:
Returns:
"""
if not auth_value:
raise ValueError(
"TuneV2AdvertiserSites: Collect: Value 'auth_value' not provided."
)
if not auth_type:
raise ValueError(
"TuneV2AdvertiserSites: Collect: Value 'auth_type' not valid.")
if not auth_type_use or \
not TuneV2AuthenticationTypes.validate(auth_type_use):
raise ValueError(
"TMC v3 Logs Advertisers Base: Collect: Value 'auth_type_use' not valid."
)
_request_params = {"source": "multiverse"}
if auth_type == TuneV2AuthenticationTypes.API_KEY:
self.api_key = auth_value
if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = tmc_auth_v2_session_token(
tmc_api_key=self.api_key,
logger_level=self.logger_level,
logger_format=self.logger_format)
_request_params.update({
"session_token": self.session_token,
})
else:
_request_params.update({
"api_key": self.api_key,
})
elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN:
self.session_token = auth_value
_request_params.update({
"session_token": self.session_token,
})
else:
raise ValueError("Invalid 'auth_type': '{}'".format(auth_type))
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller="advertiser/sites",
action="find"
)
self.logger.info("Start Advertiser Sites find")
filter_status = None
if site_status == TuneV2AdvertiserSiteStatus.ACTIVE:
filter_status = "(status = \"active\")"
elif site_status == TuneV2AdvertiserSiteStatus.NOT_ACTIVE:
filter_status = "(status != \"active\")"
filter_sites = None
if site_ids and len(site_ids) > 0:
filter_sites = "(id IN (" + ",".join(site_ids) + "))"
if filter_status and filter_sites:
_request_params["filter"] = "{} AND {}".format(
filter_status, filter_sites)
elif filter_status:
_request_params["filter"] = filter_status
elif filter_sites:
_request_params["filter"] = filter_sites
if request_params:
if "filter" in request_params and request_params["filter"]:
if "filter" in _request_params and _request_params["filter"]:
_request_params["filter"] = "({} AND {})".format(
_request_params["filter"], request_params["filter"])
else:
_request_params["filter"] = "({})".format(
request_params["filter"])
_request_params.update({'limit': request_params.get('limit', 0)})
else:
_request_params.update({'limit': 0})
response = None
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=_request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v2_request_retry_func,
request_retry_excps_func=None,
request_label="{}.{}".format(self.__class__.__name__,
sys._getframe().f_code.co_name))
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TuneV2AdvertiserSites: Collect: Failed",
extra=tmc_req_ex.to_dict(),
)
yield (None, tmc_req_ex)
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TuneV2AdvertiserSites: Collect: Failed",
extra=tmc_rep_ex.to_dict(),
)
yield (None, tmc_rep_ex)
except Exception as ex:
print_traceback(ex)
self.logger.error(get_exception_message(ex))
yield (None, ex)
if response:
json_response = response.json()
if not json_response or \
json_response['status_code'] != 200 or \
'errors' in json_response:
raise TuneReportingError(
error_message="Failed to get advertiser sites: {}, {}".
format(
json_response['status_code'],
json_response['errors'].get('message', None),
),
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
if ('data' not in json_response or not json_response['data']):
raise TuneReportingError(
error_message="Missing 'data': {}".format(
str(json_response)),
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
data = json_response['data']
else:
data = None
if not data or len(data) == 0:
yield ([], None)
else:
for (i, item) in enumerate(data):
yield (item, None)
def get_advertiser_id(self, auth_type, auth_value, request_retry=None):
"""Get Advertiser ID
Args:
auth_type:
auth_value:
request_retry:
Returns:
"""
if not auth_type:
raise ValueError(
"TMC v2 Advertisers: Get Advertiser ID: Value 'auth_type' not provided."
)
if not auth_value:
raise ValueError(
"TMC v2 Advertisers: Get Advertiser ID: Value 'auth_value' not provided."
)
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller="advertiser",
action="find"
)
request_params = {auth_type: auth_value}
self.logger.info("TMC v2 Advertisers: Advertiser ID")
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v2_request_retry_func,
request_retry_excps_func=None,
request_label="TMC v2 Advertisers")
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Advertisers: Advertiser ID: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Advertisers: Advertiser ID: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v2 Advertisers: Advertiser ID: Failed",
extra={'error': get_exception_message(ex)},
)
raise TuneReportingError(
error_message=("TMC v2 Advertisers: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Advertisers: Advertiser ID:")
self.logger.debug("TMC v2 Advertisers: Advertiser ID",
extra={'response': json_response})
json_response_status_code = json_response['status_code']
http_status_successful = is_http_status_type(
http_status_code=json_response_status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
if not http_status_successful or not json_response['data']:
raise TuneReportingError(
error_message="TMC v2 Advertisers: Failed: {}".format(
json_response_status_code),
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
if 'data' not in json_response or \
not json_response['data'] or \
len(json_response['data']) == 0:
raise TuneReportingError(
error_message="TMC v2 Advertisers: Advertiser ID: Failed",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
advertiser_data = json_response['data'][0]
self.advertiser_id = advertiser_data.get('id', None)
if self.advertiser_id is None:
raise TuneReportingError(
error_message="TMC v2 Advertisers: Advertiser ID: Failed",
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
self.logger.info("TMC v2 Advertisers: Advertiser ID: {}".format(
self.advertiser_id))
return True
def get_session_token(self, tmc_api_key, request_retry=None):
"""Generate session token is returned to provide
access to service.
Args:
tmc_api_key:
request_retry:
Returns:
"""
self.logger.info("TMC v2 Session Authenticate: Get Token")
self.api_key = tmc_api_key
request_url = \
self.tune_mat_request_path(
mat_api_version="v2",
controller="session/authenticate",
action="api_key"
)
request_params = \
{
'api_keys': tmc_api_key,
"source": "multiverse"
}
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v2_request_retry_func,
request_retry_excps_func=None,
request_label="TMC v2 Session Authenticate")
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v2 Session Authenticate: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v2 Session Authenticate: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v2 Session Authenticate: Failed: {}".format(
get_exception_message(ex)))
raise TuneReportingError(
error_message=(
"TMC v2 Session Authenticate: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v2 Get Session Token: Validate",
)
self.logger.debug(
"TMC v2 Session Authenticate: Details:",
extra={'response': json_response},
)
response_status_code = json_response.get('status_code', None)
status_code_successful = is_http_status_type(
http_status_code=response_status_code,
http_status_type=HttpStatusType.SUCCESSFUL)
response_errors = json_response.get('errors', None)
response_error_message = ""
if response_errors:
if isinstance(response_errors, dict):
error_message = response_errors.get('message', None)
if error_message:
if error_message.startswith("Invalid api key"):
response_status_code = TuneReportingErrorCodes.UNAUTHORIZED
response_error_message += error_message
elif isinstance(response_errors, list):
for response_error in response_errors:
if isinstance(response_error, dict) \
and 'message' in response_error:
error_message = response_error.get('message', None)
if error_message:
if error_message.startswith("Invalid api key"):
response_status_code = TuneReportingErrorCodes.UNAUTHORIZED
response_error_message += error_message
error_code = response_status_code
if not status_code_successful or not json_response['data']:
raise TuneReportingError(
error_message="TMC v2 Session Authenticate: Failed: {}".format(
response_error_message),
error_code=error_code,
)
self.session_token = json_response['data']
self.logger.info("TMC v2 Session Authenticate",
extra={'session_token': self.session_token})
self.logger.info("TMC v2 Session Authenticate: Finished")
return True
def tmc_auth_v2_advertiser(tmc_api_key,
logger_level=logging.NOTSET,
logger_format=TuneLoggingFormat.JSON):
"""TMC Authentication
:return:
"""
log.info("TMC v2 Advertiser: Authentication: Start")
response_auth = None
tune_v2_advertisers = \
TuneV2Advertisers(
logger_level=logger_level,
logger_format=logger_format
)
try:
try:
if tune_v2_advertisers.get_advertiser_id(
auth_type=TuneV2AuthenticationTypes.API_KEY,
auth_value=tmc_api_key,
request_retry=None):
advertiser_id = tune_v2_advertisers.advertiser_id
log.debug("TMC v2 Advertiser: {}".format(advertiser_id))
except TuneRequestBaseError as tmc_req_ex:
pprint(tmc_req_ex.to_dict())
print(str(tmc_req_ex))
except TuneReportingError as tmc_rep_ex:
pprint(tmc_rep_ex.to_dict())
print(str(tmc_rep_ex))
except Exception as ex:
print_traceback(ex)
print(get_exception_message(ex))
except AuthenticationError as auth_ex:
log.error("TMC v2 Advertiser: Authentication: Failed",
extra=auth_ex.to_dict())
raise TuneReportingAuthError(
error_message="TMC v2 Advertiser: Authentication: Failed",
errors=auth_ex.errors,
error_request_curl=auth_ex.request_curl,
error_code=auth_ex.remote_status,
)
except Exception as ex:
print_traceback(ex)
error_code = TuneReportingErrorCodes.REP_ERR_SOFTWARE
log.error('TMC v2 Advertiser: Authentication: Failed: Unexpected',
extra={
'exit_code': error_code,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
})
raise TuneReportingAuthError(
error_message=
"TMC v2 Advertiser: Authentication: Failed: Unexpected",
errors=ex,
error_code=error_code)
if response_auth:
log.debug("TMC v2 Advertiser: Authentication: Details",
extra=response_auth.to_dict())
return response_auth
def _collect(self,
auth_type_use,
start_date,
end_date,
request_params=None,
request_retry=None,
request_action=TuneV2AdvertiserStatsActions.FIND):
"""Collect data: TUNE Advertiser Stats Actuals.
Args:
start_date:
end_date:
request_params:
request_retry:
request_action:
Returns:
"""
self.logger.debug(("TuneV2AdvertiserStatsActuals: Collect: "
"Action: '{}'").format(request_action))
dict_request_params = self._map_request_params(auth_type_use,
start_date, end_date,
request_params)
self.logger.debug("TuneV2AdvertiserStatsActuals: Collect",
extra={'build_params': dict_request_params})
try:
if request_action == TuneV2AdvertiserStatsActions.FIND:
self._find_v2(request_params=dict_request_params,
request_retry=request_retry)
elif request_action == TuneV2AdvertiserStatsActions.EXPORT:
self._export_v2_download_csv(
auth_type_use=auth_type_use,
export_controller=self._CONTROLLER,
export_action='export',
export_status_controller=self._CONTROLLER,
export_status_action='status',
request_params=dict_request_params,
request_retry=request_retry)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Collect: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Collect: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TuneV2AdvertiserStatsActuals: Collect: {}".format(
get_exception_message(ex)))
raise TuneReportingError(
error_message=(
"TuneV2AdvertiserStatsActuals: Collect: Failed: {}"
).format(get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
def _stream_v2(
self,
auth_type_use,
start_date,
end_date,
request_params,
request_retry=None,
):
"""Stream data: TUNE Advertiser Stats Actuals.
Args:
start_date:
end_date:
request_params:
request_retry:
Returns:
"""
self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export",
extra={'request_params': request_params})
dict_request_params = self._map_request_params(auth_type_use,
start_date, end_date,
request_params)
self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export",
extra={'build_params': dict_request_params})
try:
response = self._export_stream_v2(
auth_type_use,
export_controller=self._CONTROLLER,
export_action='export',
export_status_controller=self._CONTROLLER,
export_status_action='status',
request_params=dict_request_params,
request_retry=request_retry)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Stream: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TuneV2AdvertiserStatsActuals: Stream: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TuneV2AdvertiserStatsActuals: {}".format(
get_exception_message(ex)))
raise TuneReportingError(
error_message=(
"TuneV2AdvertiserStatsActuals: Failed: {}").format(
get_exception_message(ex)),
errors=ex,
error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
return response
def _export_v3_job_to_queue(self, request_params, request_retry=None, request_label="TMC v3 Job On Queue"):
"""Gather data using action export.json.
:param request_params:
:param request_retry:
:return:
"""
request_url = \
self.tune_mat_request_path(
mat_api_version="v3",
controller=self.controller,
action="exports/{}".format(
self.logs_advertisers_type
)
)
self.logger.debug(
msg=("TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'exports': "
"Start Advertiser Report Actuals").format(self.logs_advertisers_type),
extra={'start_date': request_params["start_date"],
'end_date': request_params["end_date"]}
)
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=request_retry,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v3_request_retry_func,
request_retry_excps_func=None,
request_label=request_label
)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC v3 Logs Advertisers Base: {}".format(get_exception_message(ex)))
raise
json_response = validate_json_response(
response,
request_curl=self.mv_request.built_request_curl,
request_label="TMC v3 Logs Advertisers: '{}': Action 'exports'".format(self.logs_advertisers_type)
)
if ('handle' not in json_response or not json_response['handle']):
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'exports': "
"Response missing 'handle': {}"
).format(self.logs_advertisers_type, str(json_response)),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE
)
export_job = json_response['handle']
if not export_job:
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Logs '{}': "
"Action 'exports': "
"Response missing 'handle'"
).format(self.logs_advertisers_type),
error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE
)
self.logger.info(
"TuneV3LogsAdvertisersBase",
extra={
'advertiser_type': self.logs_advertisers_type,
'action': 'exports',
'start_date': request_params["start_date"],
'end_date': request_params["end_date"],
'job': export_job
}
)
return export_job
def _check_v3_job_status_on_queue(self, export_job, request_retry=None, request_label="TMC v3 Job Status On Queue"):
"""Status of Export Report.
Args:
export_job:
request_retry:
Returns:
"""
request_label = "v3 Logs Advertisers Check Export Status"
request_url = \
self.tune_mat_request_path(
mat_api_version="v3",
controller=self.controller,
action="exports/{}".format(
export_job
)
)
self.logger.info((
"TMC v3 Logs Advertisers Base: Check Export Status: "
"Logs '{}': "
"Action: 'exports status', "
"Status of Export Report for "
"Job Handle: '{}'"
).format(self.logs_advertisers_type, export_job))
tries = -1 # default: -1 (indefinite)
delay = 10
jitter = 0
max_delay = 60
if request_retry:
if 'delay' in request_retry:
delay = request_retry['delay']
if 'jitter' in request_retry:
jitter = request_retry['jitter']
if 'max_delay' in request_retry:
max_delay = request_retry['max_delay']
request_params = {"session_token": self.session_token}
self.logger.debug("TMC v3 Logs Advertisers Base: Check Export Status", extra={'request_url': request_url})
self.logger.debug(
"TMC v3 Logs Advertisers Base: Check Export Status: Request Retry",
extra={'tries': tries,
'delay': delay,
'jitter': jitter,
'max_delay': max_delay}
)
self.logger.debug(
"TMC v3 Logs Advertisers Base: Check Export Status: Request",
extra={'request_params': safe_dict(request_params)}
)
report_url = None
_attempts = 1
export_percent_complete = 0
export_status_action = 'exports status'
self.logger.warning(
"TMC v3 Logs Advertisers Base: Check Export Status",
extra={'job': export_job,
'attempt': _attempts,
'action': export_status_action}
)
time.sleep(10)
_tries, _delay = tries, delay
while True:
try:
response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=self.tune_v3_request_retry_func,
request_retry_excps_func=None,
request_label=request_label
)
except TuneRequestBaseError as tmc_req_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra=tmc_req_ex.to_dict(),
)
raise
except TuneReportingError as tmc_rep_ex:
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra=tmc_rep_ex.to_dict(),
)
raise
except Exception as ex:
print_traceback(ex)
self.logger.error(
"TMC v3 Logs Advertisers Base: Check Export Status: Failed",
extra={'error': get_exception_message(ex)}
)
raise
http_status_successful = is_http_status_type(
http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
if not http_status_successful:
raise TuneReportingError(
error_message="Failed to get export status on queue: {}".format(response.status_code),
error_code=TuneReportingErrorCodes.REP_ERR_REQUEST
)
json_response = response.json()
export_percent_complete = 0
if "percent_complete" in json_response:
export_percent_complete = \
safe_int(json_response["percent_complete"])
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status",
extra={
'job': export_job,
'response_status_code': json_response["status"],
'export_percent_complete': safe_int(export_percent_complete)
}
)
if (export_percent_complete == 100 and json_response["status"] == "complete" and json_response["url"]):
report_url = json_response["url"]
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status: Completed",
extra={
'job': export_job,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': safe_int(export_percent_complete)
}
)
break
if tries >= 0:
_tries -= 1
if _tries == 0:
self.logger.error(("TMC v3 Logs Advertisers Base: "
"Check Job Export Status: Exhausted Retries"),
extra={
'attempt': _attempts,
'tries': _tries,
'action': export_status_action,
'request_label': request_label,
'export_percent_complete': export_percent_complete
})
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: "
"Check Job Export Status: "
"Exhausted Retries: "
"Percent Completed: {}"
).format(export_percent_complete),
error_code=TuneReportingErrorCodes.REP_ERR_JOB_STOPPED
)
_attempts += 1
self.logger.warning(
"TMC v3 Logs Advertisers Base: Check Export Status",
extra={'attempt': _attempts,
'job': export_job,
'delay': _delay,
'action': 'exports status'}
)
time.sleep(_delay)
_delay += jitter
_delay = min(_delay, max_delay)
if export_percent_complete == 100 and not report_url:
raise TuneReportingError(
error_message=(
"TMC v3 Logs Advertisers Base: Check Job Export Status: "
"Download report URL: Undefined"
)
)
self.logger.info(
"TMC v3 Logs Advertisers Base: Check Job Export Status: Finished",
extra={
'attempt': _attempts,
'action': export_status_action,
'report_url': report_url,
'request_label': request_label,
'export_percent_complete': export_percent_complete,
'job': export_job
}
)
return report_url
def main(tmc_api_key):
TIMEZONE_COLLECT = "America/New_York"
tune_v2_advertiser_stats_actuals = \
TuneV2AdvertiserStatsActuals(
timezone=TIMEZONE_COLLECT,
logger_level=logging.INFO,
logger_format=TuneLoggingFormat.JSON
)
dw_file_path = "data.{}".format(TuneV2AdvertiserStatsFormats.JSON)
if os.path.exists(dw_file_path):
os.remove(dw_file_path)
tz = pytz.timezone(TIMEZONE_COLLECT)
yesterday = datetime.now(tz).date() - timedelta(days=1)
str_yesterday = str(yesterday)
request_params = {
'format':
TuneV2AdvertiserStatsFormats.CSV,
'fields': ("ad_clicks,"
"ad_clicks_unique,"
"ad_impressions,"
"ad_impressions_unique,"
"ad_network_id,"
"advertiser_id,"
"country.code,"
"date_hour,"
"events,"
"installs,"
"is_reengagement,"
"payouts,"
"publisher_id,"
"publisher_sub_ad.ref,"
"publisher_sub_adgroup.ref,"
"publisher_sub_campaign.ref,"
"publisher_sub_publisher.ref,"
"publisher_sub_site.ref,"
"site_id"),
'group': ("country_id,"
"is_reengagement,"
"publisher_id,"
"publisher_sub_ad_id,"
"publisher_sub_adgroup_id,"
"publisher_sub_campaign_id,"
"publisher_sub_publisher_id,"
"publisher_sub_site_id,"
"site_id"),
'timezone':
"America/Los_Angeles"
}
try:
tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key)
response = tune_v2_advertiser_stats_actuals.stream(
auth_value=tmc_api_key,
auth_type=TuneV2AuthenticationTypes.API_KEY,
auth_type_use=TuneV2AuthenticationTypes.API_KEY,
start_date=str_yesterday,
end_date=str_yesterday,
request_params=request_params,
request_retry={
'delay': 15,
'timeout': 30,
'tries': 5
})
line_count = 0
csv_keys_list = None
json_keys_dict = {
"publisher_sub_campaign.ref": "sub_campaign_ref",
"publisher_sub_ad.ref": "sub_ad_ref",
"publisher_sub_adgroup.ref": "sub_adgroup_ref",
"publisher_sub_publisher.ref": "sub_publisher_ref",
"publisher_sub_site.ref": "sub_site_ref",
"publisher_sub_placement.ref": "sub_placement_ref",
"publisher_sub_campaign.name": "sub_campaign_name",
"publisher_sub_ad.name": "sub_ad_name",
"publisher_sub_adgroup.name": "sub_adgroup_name",
"publisher_sub_publisher.name": "sub_publisher_name",
"publisher_sub_site.name": "sub_site_name",
"publisher_sub_placement.name": "sub_placement_name",
"publisher_sub_campaign_id": "sub_campaign_id",
"publisher_sub_ad_id": "sub_ad_id",
"publisher_sub_adgroup_id": "sub_adgroup_id",
"publisher_sub_publisher_id": "sub_publisher_id",
"publisher_sub_site_id": "sub_site_id",
"publisher_sub_placement_id": "publisher_sub_placement_id",
"country.code": "country_code",
"ad_impressions": "received_impressions_gross",
"ad_impressions_unique": "received_impressions_unique",
"ad_clicks": "received_clicks_gross",
"ad_clicks_unique": "received_clicks_unique",
"events": "received_engagements",
"installs": "received_installs",
"payouts": "cost"
}
json_types_dict = {
"client_id": int,
"partner_id": int,
"vendor_id": int,
"date": str,
"hour": int,
"timezone": str,
"granularity": str,
"site_ref_id": str,
"site_ref_type": str,
"partner_ref_id": int,
"partner_ref_type": str,
"partner_vendor_ref_id": int,
"partner_vendor_ref_type": str,
"sub_campaign_type": str,
"sub_campaign_ref": str,
"sub_ad_ref": str,
"sub_adgroup_ref": str,
"sub_publisher_ref": str,
"sub_site_ref": str,
"sub_placement_ref": str,
"sub_campaign_name": str,
"sub_ad_name": str,
"sub_adgroup_name": str,
"sub_publisher_name": str,
"sub_site_name": str,
"sub_placement_name": str,
"sub_campaign_name": str,
"sub_ad_name": str,
"sub_adgroup_name": str,
"sub_publisher_name": str,
"sub_site_name": str,
"sub_placement_name": str,
"sub_campaign_id": int,
"sub_ad_id": int,
"sub_adgroup_id": int,
"sub_publisher_id": int,
"sub_site_id": int,
"publisher_sub_placement_id": int,
"country_code": str,
"received_impressions_gross": int,
"received_impressions_unique": int,
"received_clicks_gross": int,
"received_clicks_unique": int,
"received_installs": int,
"received_engagements": int,
"received_conversions": int,
"cost": float,
"cost_currency": str,
"site_id": int,
"publisher_id": int,
"advertiser_id": int,
"ad_network_id": int,
"ad_impressions": int,
}
client_id = 0
partner_id = 0
vendor_id = 0
timezone = "TBD"
granularity = "TBD"
config_extra = {
"client_id": client_id,
"partner_id": partner_id,
"vendor_id": vendor_id,
"timezone": timezone,
"granularity": granularity,
"cost_currency": "USD",
"received_conversions": 0,
"site_ref_type": "tmc",
"partner_ref_type": "tmc",
"partner_vendor_ref_type": "tmc"
}
with open(file=dw_file_path, mode='w') as dw_file_w:
for bytes_line in response.iter_lines(chunk_size=4096):
if bytes_line: # filter out keep-alive new chunks
line_count += 1
str_line = bytes_line.decode("utf-8")
if line_count == 1:
csv_keys_list = str_line.split(',')
for index, csv_key in enumerate(csv_keys_list):
if csv_key in json_keys_dict:
csv_keys_list[index] = json_keys_dict[csv_key]
continue
elif line_count > 2:
dw_file_w.write('\n')
csv_values_list = str_line.split(',')
json__dict = {}
is_reengagement = 0
received_installs = 0
received_engagements = 0
for csv_key, csv_value in zip(csv_keys_list,
csv_values_list):
csv_value_strip = csv_value.strip('"')
if csv_key == "date_hour":
parts_date_time = csv_value_strip.split(" ")
rdate_yyyy_mm_dd = parts_date_time[0]
parts_time = parts_date_time[1].split(":")
rhour = safe_int(parts_time[0])
json__dict.update({"date": rdate_yyyy_mm_dd})
json__dict.update({"hour": rhour})
elif csv_key == "is_reengagement":
is_reengagement = safe_int(csv_value_strip)
elif csv_key == "received_installs":
received_installs = safe_int(csv_value_strip)
json__dict.update(
{'received_installs': received_installs})
elif csv_key == "received_engagements":
received_engagements = safe_int(csv_value_strip)
json__dict.update(
{'received_engagements': received_engagements})
else:
if csv_key in json_types_dict:
if json_types_dict[csv_key] == str:
csv_value_typed = safe_str(csv_value_strip)
elif json_types_dict[csv_key] == int:
csv_value_typed = safe_int(csv_value_strip)
elif json_types_dict[csv_key] == float:
csv_value_typed = safe_float(
csv_value_strip)
else:
csv_value_typed = safe_str(csv_value_strip)
else:
csv_value_typed = safe_str(csv_value_strip)
json__dict.update({csv_key: csv_value_typed})
if is_reengagement == 1:
engagements = received_engagements
else:
engagements = 0
if engagements > 0 and received_installs > 0:
sub_campaign_type = "acquisition_engagement"
elif received_installs > 0:
sub_campaign_type = "acquisition"
elif engagements > 0:
sub_campaign_type = "engagement"
else:
sub_campaign_type = ""
json__dict.update({'sub_campaign_type': sub_campaign_type})
json__dict.update(config_extra)
json_str = json.dumps(json__dict)
dw_file_w.write(json_str)
dw_file_w.flush()
statinfo = os.stat(dw_file_path)
extra = {
'response_status_code': response.status_code,
'response_headers': response.headers,
'dw_file_path': dw_file_path,
'dw_file_size': convert_size(statinfo.st_size),
'line_count': line_count,
'csv_header_list': csv_keys_list
}
pprint(extra)
except TuneRequestBaseError as tmc_req_ex:
print_traceback(tmc_req_ex)
pprint(tmc_req_ex.to_dict())
print(str(tmc_req_ex))
except TuneReportingError as tmc_rep_ex:
pprint(tmc_rep_ex.to_dict())
print(str(tmc_rep_ex))
except Exception as ex:
print_traceback(ex)
print(get_exception_message(ex))
def main(tmc_api_key):
tune_v2_advertiser_stats_actuals = \
TuneV2AdvertiserStatsActuals(
logger_level=logging.INFO,
logger_format=TuneLoggingFormat.JSON
)
tz = pytz.timezone("America/New_York")
yesterday = datetime.now(tz).date() - timedelta(days=1)
str_yesterday = str(yesterday)
try:
auth_response = tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key)
assert auth_response
tune_v2_advertiser_stats_actuals.collect(
auth_value=tmc_api_key,
auth_type=TuneV2AuthenticationTypes.API_KEY,
auth_type_use=TuneV2AuthenticationTypes.API_KEY,
start_date=str_yesterday,
end_date=str_yesterday,
request_params={
'timezone': 'America/Los_Angeles',
'format': TuneV2AdvertiserStatsFormats.CSV,
'fields': (
"ad_clicks,"
"ad_clicks_unique,"
"ad_impressions,"
"ad_impressions_unique,"
"ad_network_id,"
"advertiser_id,"
"country.code,"
"date_hour,"
"events,"
"installs,"
"is_reengagement,"
"payouts,"
"publisher_id,"
"publisher_sub_ad.ref,"
"publisher_sub_adgroup.ref,"
"publisher_sub_campaign.ref,"
"publisher_sub_publisher.ref,"
"publisher_sub_site.ref,"
"site_id"
),
'group': (
"country_id,"
"is_reengagement,"
"publisher_id,"
"publisher_sub_ad_id,"
"publisher_sub_adgroup_id,"
"publisher_sub_campaign_id,"
"publisher_sub_publisher_id,"
"publisher_sub_site_id,"
"site_id"
),
'timezone': "America/Los_Angeles",
'limit': 5
},
request_action=TuneV2AdvertiserStatsActions.EXPORT,
request_retry={'delay': 15,
'timeout': 30,
'tries': 10}
)
except TuneRequestBaseError as tmc_req_ex:
print_traceback(tmc_req_ex)
pprint(tmc_req_ex.to_dict())
print(str(tmc_req_ex))
except TuneReportingError as tmc_rep_ex:
pprint(tmc_rep_ex.to_dict())
print(str(tmc_rep_ex))
except Exception as ex:
print_traceback(ex)
print(get_exception_message(ex))
for row in list(tune_v2_advertiser_stats_actuals.generator):
pprint(row)
def tmc_auth(self, tmc_api_key):
"""TMC Authentication"""
if not tmc_api_key:
raise ValueError("Parameter 'tmc_api_key' not defined.")
self.logger.info("TMC Authentication: Start")
# Attempt to authenticate.
request_url = ('https://api.mobileapptracking.com/v2/advertiser/find')
request_params = {'api_key': tmc_api_key}
auth_request_curl = command_line_request_curl_get(request_url=request_url, request_params=request_params)
try:
auth_response = self.mv_request.request(
request_method="GET",
request_url=request_url,
request_params=request_params,
request_retry=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_label="TMC Authentication"
)
except TuneRequestBaseError as tmc_ex:
self.logger.error("TMC Authentication: Failed", extra=tmc_ex.to_dict())
raise
except Exception as ex:
print_traceback(ex)
self.logger.error("TMC Authentication: Failed: {}".format(get_exception_message(ex)))
raise
if auth_response.status_code != requests.codes.ok:
raise TuneReportingError(
error_message="Invalid request",
error_request_curl=auth_request_curl,
error_code=auth_response.status_code
)
try:
decoded_resp = auth_response.json()
except Exception as ex:
# No JSON response available.
raise TuneReportingError(
error_message='Invalid JSON response: {}'.format(auth_response.text),
errors=ex,
error_request_curl=auth_request_curl,
error_code=TuneReportingErrorCodes.REP_ERR_AUTH_JSON_ERROR
)
tmc_status_code = decoded_resp.get('status_code', None)
tmc_errors = decoded_resp.get('errors', None)
if tmc_errors:
error_code = tmc_status_code
errors = []
if isinstance(tmc_errors, list):
error_list = tmc_errors
error_list = error_list if error_list else []
for error in error_list:
error_message = error.get('message', None)
if error_message:
errors.append(error_message)
if error_message.startswith("Invalid api key"):
error_code = TuneReportingErrorCodes.UNAUTHORIZED
elif isinstance(tmc_errors, dict):
error_message = tmc_errors.get('message', None)
if error_message:
errors.append(error_message)
raise TuneReportingError(
error_message="Error status: {}".format(tmc_errors),
error_request_curl=auth_request_curl,
error_code=error_code,
)
return decoded_resp