def main(tmc_api_key): tune_v3_logs_advertisers_impressions_export = \ TuneV3LogsAdvertisersImpressions( logger_level=logging.INFO, logger_format=LoggingFormat.JSON, logger_output=LoggingOutput.STDOUT_COLOR ) tune_v3_logs_advertisers_impressions_export.logger_level = logging.DEBUG tz = pytz.timezone("America/New_York") yesterday = datetime.now(tz).date() - timedelta(days=1) str_yesterday = str(yesterday) try: tune_v3_logs_advertisers_impressions_export.collect( auth_value=tmc_api_key, auth_type=TuneV2AuthenticationTypes.API_KEY, auth_type_use=TuneV2AuthenticationTypes.SESSION_TOKEN, start_date=str_yesterday, end_date=str_yesterday, advertiser_id=877, request_params={'timezone': 'America/New_York', 'filter': "(ad_network_id <> 0)", 'limit': 5}, request_action=TuneV3LogsAdvertisersActions.EXPORT ) except TuneRequestBaseError as tmc_req_ex: print_traceback(tmc_req_ex) pprint(tmc_req_ex.to_dict()) print(str(tmc_req_ex)) except TuneReportingError as tmc_rep_ex: pprint(tmc_rep_ex.to_dict()) print(str(tmc_rep_ex)) except Exception as ex: print_traceback(ex) print(get_exception_message(ex)) for row in list(tune_v3_logs_advertisers_impressions_export.generator): pprint(row)
def tmc_auth_v2_session_token( tmc_api_key, logger_level=logging.NOTSET, logger_format=TuneLoggingFormat.JSON, ): """TMC Authentication :return: """ log.info("TMC v2 Session Token: Request") try: tune_v2_session_authenticate = \ TuneV2SessionAuthenticate( logger_level=logger_level, logger_format=logger_format ) if tune_v2_session_authenticate.get_session_token(tmc_api_key=tmc_api_key, request_retry=None): session_token = tune_v2_session_authenticate.session_token except TuneReportingError as auth_ex: log.error("TMC v2 Session Token: Failed", extra=auth_ex.to_dict()) raise except Exception as ex: print_traceback(ex) log.error( 'TMC v2 Session Token: Failed: Unexpected', extra={'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex)} ) raise TuneReportingAuthError( error_message="TMC v2 Session Token: Failed: Unexpected", errors=ex, ) log.debug("TMC v2 Session Token: Response", extra={'session_token': session_token}) return session_token
def _find_v3(self, request_params, request_retry=None, request_label="TMC v3 Logs Advertisers"): """Gather data using action 'find' Args: request_params: request_retry: Returns: """ self.logger.debug(("TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'find'").format(self.logs_advertisers_type)) if "start_date" not in request_params: raise ValueError("Missing attribute 'start_date' in parameter 'request_params'") request_start_date = request_params["start_date"] if "end_date" not in request_params: raise ValueError("Missing attribute 'end_date' in parameter 'request_params'") request_end_date = request_params["end_date"] datetime_start = dt.datetime.strptime(request_start_date, '%Y-%m-%d') datetime_end = dt.datetime.strptime(request_end_date, '%Y-%m-%d') str_date_start = str(datetime_start.date()) str_date_end = str(datetime_end.date()) self.logger.debug( "TMC v3 Logs Advertisers Base: Action 'find'", extra={'start_date': str_date_start, 'end_date': str_date_end} ) if not request_retry: request_retry = {'timeout': 600} request_url = \ self.tune_mat_request_path( mat_api_version="v3", controller=self.controller, action=self.logs_advertisers_type ) if 'response_format' not in request_params: request_params['response_format'] = \ TuneV3LogsAdvertisersResponseFormats.JSON request_params["start_date"] += "T00:00:00Z" request_params["end_date"] += "T23:59:59Z" try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_label=request_label ) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v3 Logs Advertisers Base: {}".format(get_exception_message(ex))) raise json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v3 Logs Advertisers: Action 'find'", ) data = json_response['data'] self.data = data
def main(tmc_api_key): TIMEZONE_COLLECT = "America/New_York" export_format = TuneV2AdvertiserStatsFormats.CSV tune_v2_advertiser_stats_actuals = \ TuneV2AdvertiserStatsActuals( timezone=TIMEZONE_COLLECT, logger_level=logging.INFO, logger_format=LoggingFormat.JSON, logger_output=LoggingOutput.STDOUT_COLOR ) dw_file_path = "data.{}".format(export_format) if os.path.exists(dw_file_path): os.remove(dw_file_path) tz = pytz.timezone(TIMEZONE_COLLECT) yesterday = datetime.now(tz).date() - timedelta(days=1) str_yesterday = str(yesterday) try: tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key) response = tune_v2_advertiser_stats_actuals.stream( auth_value=tmc_api_key, auth_type=TuneV2AuthenticationTypes.API_KEY, auth_type_use=TuneV2AuthenticationTypes.API_KEY, start_date=str_yesterday, end_date=str_yesterday, request_params={ 'format': export_format, 'fields': ( "ad_clicks," "ad_clicks_unique," "ad_impressions," "ad_impressions_unique," "ad_network_id," "advertiser_id," "country.code," "date_hour," "events," "installs," "is_reengagement," "payouts," "publisher_id," "publisher_sub_ad.ref," "publisher_sub_adgroup.ref," "publisher_sub_campaign.ref," "publisher_sub_publisher.ref," "publisher_sub_site.ref," "site_id" ), 'group': ( "country_id," "is_reengagement," "publisher_id," "publisher_sub_ad_id," "publisher_sub_adgroup_id," "publisher_sub_campaign_id," "publisher_sub_publisher_id," "publisher_sub_site_id," "site_id" ), 'timezone': "America/Los_Angeles" }, request_retry={'delay': 15, 'timeout': 30, 'tries': 10} ) with open(file=dw_file_path, mode='wb') as dw_file_wb: for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks dw_file_wb.write(chunk) dw_file_wb.flush() with open(file=dw_file_path, mode='r') as csv_file_r: csv_dict_reader = csv.DictReader(csv_file_r) for row in csv_dict_reader: print(row) statinfo = os.stat(dw_file_path) extra = { 'response_status_code': response.status_code, 'response_headers': response.headers, 'dw_file_path': dw_file_path, 'dw_file_size': convert_size(statinfo.st_size) } pprint(extra) except TuneRequestBaseError as tmc_req_ex: print_traceback(tmc_req_ex) pprint(tmc_req_ex.to_dict()) print(str(tmc_req_ex)) except TuneReportingError as tmc_rep_ex: pprint(tmc_rep_ex.to_dict()) print(str(tmc_rep_ex)) except Exception as ex: print_traceback(ex) print(get_exception_message(ex))
def _check_v2_job_status_on_queue( self, auth_type, auth_value, export_status_controller, export_status_action, export_job_id, request_retry=None, ): """Check Job Export Status Args: export_status_controller: export_status_action: export_job_id: request_retry: Returns: """ request_label = "TMC v2 Advertiser Stats: Check Export Status" v2_export_status_request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller=export_status_controller, action=export_status_action ) request_params = {auth_type: auth_value, "job_id": export_job_id} self.logger.info( ("TMC v2 Advertiser Stats: Check Job Status"), extra={ 'action': export_status_action, 'job_id': export_job_id, 'request_url': v2_export_status_request_url, 'request_params': safe_dict(request_params) }) tries = 60 # -1 (indefinite) delay = 10 jitter = 10 max_delay = 60 if request_retry is not None: if 'delay' in request_retry: delay = request_retry['delay'] if 'jitter' in request_retry: jitter = request_retry['jitter'] if 'max_delay' in request_retry: max_delay = request_retry['max_delay'] if 'tries' in request_retry: tries = request_retry['tries'] else: request_retry.update({'tries': 60}) else: request_retry = {'tries': 60, 'delay': 10, 'timeout': 60} self.logger.debug(msg=("TMC v2 Advertiser Stats: Check Job Status: " "Request Retry"), extra=request_retry) report_url = None _attempts = 1 export_percent_complete = 0 time.sleep(10) _tries, _delay = tries, delay while True: try: response = self.mv_request.request( request_method="GET", request_url=v2_export_status_request_url, request_params=request_params, request_label=request_label, request_retry_func=self.tune_v2_request_retry_func) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertiser Stats: Check Job Status: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertiser Stats: Check Job Status: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TMC v2 Advertiser Stats: Check Job Status: {}".format( get_exception_message(ex))) raise http_status_successful = is_http_status_type( http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL) if not http_status_successful: raise TuneReportingError( error_message=( "Failed to get export status on queue: {}").format( response.status_code), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST) if hasattr(response, 'url'): self.logger.info( "TMC v2 Advertiser Stats: Reporting API: Status URL", extra={'response_url': response.url}) json_response = response.json() if not json_response: request_status_successful = False elif 'status_code' not in json_response: request_status_successful = False else: status_code = json_response['status_code'] request_status_successful = is_http_status_type( http_status_code=status_code, http_status_type=HttpStatusType.SUCCESSFUL) errors = None if 'errors' in json_response: errors = json_response['errors'] if not request_status_successful: error_message = ( "TMC v2 Advertiser Stats: Check Job Status: GET '{}', Failed: {}, {}" ).format(v2_export_status_request_url, status_code, errors) if (status_code == TuneReportingError.EX_SRV_ERR_500_INTERNAL_SERVER): self.logger.error(error_message) elif (status_code == TuneReportingError.EX_SRV_ERR_503_SERVICE_UNAVAILABLE): self.logger.error(error_message) elif (status_code == TuneReportingError.EX_SRV_ERR_504_SERVICE_TIMEOUT): self.logger.error(error_message) continue elif (status_code == TuneReportingError.EX_CLT_ERR_408_REQUEST_TIMEOUT): self.logger.error( "GET '{}' request timeout, Retrying: {}".format( v2_export_status_request_url, status_code)) continue else: raise TuneReportingError(error_message=error_message, error_code=status_code) if tries >= 0 and _tries <= 1: if (status_code == HttpStatusCode.GATEWAY_TIMEOUT): raise TuneReportingError( error_message=error_message, error_code=TuneReportingErrorCodes.GATEWAY_TIMEOUT) elif (status_code == HttpStatusCode.REQUEST_TIMEOUT): raise TuneReportingError( error_message=error_message, error_code=TuneReportingErrorCodes.REQUEST_TIMEOUT) else: raise TuneReportingError(error_message=error_message, error_code=status_code) else: self.logger.warning(error_message) export_percent_complete = 0 if 'data' in json_response and json_response['data']: json_data = json_response['data'] if "percent_complete" in json_data: export_percent_complete = \ safe_int(json_data["percent_complete"]) self.logger.info(msg=("TMC v2 Advertiser Stats: " "Check Job Export Status: " "Response Success"), extra={ 'job_id': export_job_id, 'export_status': json_data["status"], 'export_percent_complete': safe_int(export_percent_complete), 'attempt': _attempts }) if (export_status_action == TuneV2AdvertiserStatsStatusAction.STATUS): if (export_percent_complete == 100 and json_data["status"] == "complete" and json_data["url"]): report_url = json_data["url"] self.logger.debug( ("TMC v2 Advertiser Stats: " "Check Job Export Status: Completed"), extra={ 'job_id': export_job_id, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label }) break elif (export_status_action == TuneV2AdvertiserStatsStatusAction.DOWNLOAD): if (export_percent_complete == 100 and json_data["status"] == "complete" and json_data["data"]["url"]): report_url = json_data["data"]["url"] self.logger.debug( ("TMC v2 Advertiser Stats: " "Check Job Export Status: Completed"), extra={ 'job_id': export_job_id, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label }) break else: self.logger.debug("TMC v2 Advertiser Stats: " "Check Job Export Status: " "No Data Available") if tries >= 0: _tries -= 1 if _tries == 0: self.logger.error( ("TMC v2 Advertiser Stats: " "Check Job Export Status: Exhausted Retries"), extra={ 'attempt': _attempts, 'tries': _tries, 'action': export_status_action, 'request_label': request_label, 'export_percent_complete': safe_int(export_percent_complete), 'job_id': export_job_id }) raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Check Job Export Status: " "Exhausted Retries: " "Percent Completed: {}").format( safe_int(export_percent_complete)), error_code=TuneReportingErrorCodes. REP_ERR_RETRY_EXHAUSTED) _attempts += 1 self.logger.info("TMC v2 Advertiser Stats: Check Job Status", extra={ 'attempt': _attempts, 'job_id': export_job_id, 'delay': _delay, 'action': export_status_action }) time.sleep(_delay) _delay += jitter _delay = min(_delay, max_delay) if export_percent_complete == 100 and not report_url: raise TuneReportingError( error_message=( "TMC v2 Advertiser Stats: Check Job Export Status: " "Download report URL: Undefined"), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) self.logger.info( "TMC v2 Advertiser Stats: Check Job Export Status: Finished", extra={ 'attempt': _attempts, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label, 'export_percent_complete': export_percent_complete, 'job_id': export_job_id }) return report_url
def _export_v2_job_to_queue( self, export_controller, export_action, request_params, request_retry=None, request_label="TMC v2 Job To Queue", ): """Export Report Request to Job Queue Args: export_controller: export_action: request_params: request_retry: Returns: Export Job ID """ request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller=export_controller, action=export_action ) self.logger.debug( ("TMC v2 Advertiser Stats: " "Start Advertiser Report Actuals"), extra={ 'action': export_action, 'start_date': request_params["start_date"], 'end_date': request_params["end_date"] }) try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_label=request_label, request_retry_func=self.tune_v2_request_retry_func) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertiser Stats: Request Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertiser Stats: Reporting Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error(get_exception_message(ex)) raise TuneReportingError( error_message=( "TMC v2 Advertiser Stats: Unexpected Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Advertiser Stats: Action '{}'".format( export_action)) if hasattr(response, 'url'): self.logger.info( "TMC v2 Advertiser Stats: Reporting API: Export URL", extra={'response_url': response.url}) if (not json_response or json_response['status_code'] != 200 or 'errors' in json_response): raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Action '{}': " "Failed to export stats: {}, {}").format( export_action, json_response['status_code'], json_response['errors']), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST) if ('data' not in json_response or not json_response['data']): raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Action '{}': " "Missing data").format(export_action), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) json_data = json_response['data'] export_job_id = None if export_action == TuneV2AdvertiserStatsExportActions.EXPORT: if ('job_id' not in json_data or not json_data['job_id']): raise TuneReportingError(error_message=( "TMC v2 Advertiser Stats: " "Action '{}': " "Response missing 'export_job_id': {}").format( export_action, str(json_data)), error_code=TuneReportingErrorCodes. REP_ERR_UNEXPECTED_VALUE) export_job_id = json_data['job_id'] elif export_action == TuneV2AdvertiserStatsExportActions.FIND_EXPORT_QUEUE: export_job_id = json_data if not export_job_id: raise TuneReportingError( error_message=( "TMC v2 Advertiser Stats: " "Action '{}': " "Response missing 'job_id'").format(export_action), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) self.logger.info("TMC v2 Advertiser Stats: Reporting API: Job ID", extra={'job_id': export_job_id}) return export_job_id
def _find_v2( self, request_params, request_retry=None, request_label="TMC v2 Advertiser Stats Find", ): """Gather data using action find.json. """ self.logger.debug("TuneV2AdvertiserStatsBase", extra={'action': 'find'}) if "start_date" not in request_params: raise ValueError( "Missing attribute 'start_date' in parameter 'request_params'") request_start_date = request_params["start_date"] if "end_date" not in request_params: raise ValueError( "Missing attribute 'end_date' in parameter 'request_params'") request_end_date = request_params["end_date"] datetime_start = dt.datetime.strptime(request_start_date, '%Y-%m-%d') datetime_end = dt.datetime.strptime(request_end_date, '%Y-%m-%d') str_date_start = str(datetime_start.date()) str_date_end = str(datetime_end.date()) self.logger.debug( ("TuneV2AdvertiserStatsBase"), extra={ 'action': 'find', 'start_date': str_date_start, 'end_date': str_date_end }) if not request_retry: request_retry = {'timeout': 60} request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller=self.controller, action="find" ) if 'format' not in request_params: request_params['format'] = TuneV2AdvertiserStatsFormats.JSON request_params["start_date"] += " 00:00:00" request_params["end_date"] += " 23:59:59" try: response = self.mv_request.request(request_method="GET", request_url=request_url, request_params=request_params, request_label=request_label) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertiser Stats: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertiser Stats: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v2 Advertiser Stats: {}".format( get_exception_message(ex))) raise json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Advertiser Stats: Action 'find'") if json_response['status_code'] != 200: raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Action 'find': " "Failed to find stats: {}, {}").format( json_response['status_code'], json.dumps(json_response), ), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST) data = json_response['data'] self.data = data
def _collect( self, auth_type_use, start_date, end_date, request_params=None, request_retry=None, request_action=TuneV3LogsAdvertisersActions.FIND ): """Collect data: TUNE Advertiser Stats Clicks. Args: start_date: end_date: request_params: request_retry: request_action: Returns: """ auth_value = None if auth_type_use == TuneV2AuthenticationTypes.API_KEY: auth_value = self.api_key elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: auth_value = self.session_token dict_request_params = { auth_type_use: auth_value, "timezone": self.timezone, "fields": ( "created," "ad_network_id," "campaign.id," "campaign.name," "publisher.id," "publisher.name," "publisher_ref_id," "publisher_sub_site.id," "publisher_sub_site.ref," "publisher_sub_site.name," "publisher_sub_campaign.id," "publisher_sub_campaign.ref," "publisher_sub_campaign.name," "request_url," "site.id," "site.mobile_app_type," "site.name," "site.package_name," "site.store_app_id" ), "filter": "({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA), "start_date": start_date, "end_date": end_date, "debug": 0 } if request_params: if "fields" in request_params and request_params["fields"]: dict_request_params["fields"] = \ request_params["fields"] if "group" in request_params and request_params["group"]: dict_request_params["group"] = \ request_params["group"] if "timestamp" in request_params and request_params["timestamp"]: dict_request_params["timestamp"] = \ request_params["timestamp"] if "filter" in request_params and request_params["filter"]: dict_request_params["filter"] = "({} AND {})".format( request_params["filter"], self._FILTER_NOT_DEBUG_NOR_TEST_DATA ) if "limit" in request_params: dict_request_params["limit"] = \ int(request_params["limit"]) if "debug" in request_params: dict_request_params["debug"] = \ int(request_params["debug"]) timezone = None if "timezone" in request_params: timezone = request_params["timezone"] if timezone: if not validate_tz_name(timezone): return TuneReportingError(error_message="Invalid Timezone: {}".format(timezone)) self.timezone = timezone dict_request_params["timezone"] = \ self.timezone self.logger.debug(("TMC v3 Logs Advertisers Clicks: " "Action '{}', Params: {}").format(request_action, str(dict_request_params))) self.logger.debug(("TMC v3 Logs Advertisers Clicks: " "Timezone: {}").format(self.timezone)) try: if request_action == TuneV3LogsAdvertisersActions.FIND: request_params["sorts"] = "created desc" self._find_v3(request_params=dict_request_params, request_retry=request_retry) elif request_action == TuneV3LogsAdvertisersActions.EXPORT: self._export_v3_download_csv(request_params=dict_request_params, request_retry=request_retry) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Clicks: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Clicks: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v3 Logs Advertisers Clicks: Failed: {}".format(get_exception_message(ex))) raise TuneReportingError( error_message=("TMC v3 Logs Advertisers Clicks: Failed: {}").format(get_exception_message(ex)), errors=ex )
def collect(self, auth_value, auth_type, auth_type_use, site_ids=None, request_params=None, site_status=TuneV2AdvertiserSiteStatus.ACTIVE): """Collect TMC v2 Advertiser Sites Args: tmc_api_key: site_ids: request_params: request_retry: site_status: Returns: """ if not auth_value: raise ValueError( "TuneV2AdvertiserSites: Collect: Value 'auth_value' not provided." ) if not auth_type: raise ValueError( "TuneV2AdvertiserSites: Collect: Value 'auth_type' not valid.") if not auth_type_use or \ not TuneV2AuthenticationTypes.validate(auth_type_use): raise ValueError( "TMC v3 Logs Advertisers Base: Collect: Value 'auth_type_use' not valid." ) _request_params = {"source": "multiverse"} if auth_type == TuneV2AuthenticationTypes.API_KEY: self.api_key = auth_value if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = tmc_auth_v2_session_token( tmc_api_key=self.api_key, logger_level=self.logger_level, logger_format=self.logger_format) _request_params.update({ "session_token": self.session_token, }) else: _request_params.update({ "api_key": self.api_key, }) elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = auth_value _request_params.update({ "session_token": self.session_token, }) else: raise ValueError("Invalid 'auth_type': '{}'".format(auth_type)) request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller="advertiser/sites", action="find" ) self.logger.info("Start Advertiser Sites find") filter_status = None if site_status == TuneV2AdvertiserSiteStatus.ACTIVE: filter_status = "(status = \"active\")" elif site_status == TuneV2AdvertiserSiteStatus.NOT_ACTIVE: filter_status = "(status != \"active\")" filter_sites = None if site_ids and len(site_ids) > 0: filter_sites = "(id IN (" + ",".join(site_ids) + "))" if filter_status and filter_sites: _request_params["filter"] = "{} AND {}".format( filter_status, filter_sites) elif filter_status: _request_params["filter"] = filter_status elif filter_sites: _request_params["filter"] = filter_sites if request_params: if "filter" in request_params and request_params["filter"]: if "filter" in _request_params and _request_params["filter"]: _request_params["filter"] = "({} AND {})".format( _request_params["filter"], request_params["filter"]) else: _request_params["filter"] = "({})".format( request_params["filter"]) _request_params.update({'limit': request_params.get('limit', 0)}) else: _request_params.update({'limit': 0}) response = None try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=_request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v2_request_retry_func, request_retry_excps_func=None, request_label="{}.{}".format(self.__class__.__name__, sys._getframe().f_code.co_name)) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TuneV2AdvertiserSites: Collect: Failed", extra=tmc_req_ex.to_dict(), ) yield (None, tmc_req_ex) except TuneReportingError as tmc_rep_ex: self.logger.error( "TuneV2AdvertiserSites: Collect: Failed", extra=tmc_rep_ex.to_dict(), ) yield (None, tmc_rep_ex) except Exception as ex: print_traceback(ex) self.logger.error(get_exception_message(ex)) yield (None, ex) if response: json_response = response.json() if not json_response or \ json_response['status_code'] != 200 or \ 'errors' in json_response: raise TuneReportingError( error_message="Failed to get advertiser sites: {}, {}". format( json_response['status_code'], json_response['errors'].get('message', None), ), error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) if ('data' not in json_response or not json_response['data']): raise TuneReportingError( error_message="Missing 'data': {}".format( str(json_response)), error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) data = json_response['data'] else: data = None if not data or len(data) == 0: yield ([], None) else: for (i, item) in enumerate(data): yield (item, None)
def get_advertiser_id(self, auth_type, auth_value, request_retry=None): """Get Advertiser ID Args: auth_type: auth_value: request_retry: Returns: """ if not auth_type: raise ValueError( "TMC v2 Advertisers: Get Advertiser ID: Value 'auth_type' not provided." ) if not auth_value: raise ValueError( "TMC v2 Advertisers: Get Advertiser ID: Value 'auth_value' not provided." ) request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller="advertiser", action="find" ) request_params = {auth_type: auth_value} self.logger.info("TMC v2 Advertisers: Advertiser ID") try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v2_request_retry_func, request_retry_excps_func=None, request_label="TMC v2 Advertisers") except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertisers: Advertiser ID: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertisers: Advertiser ID: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TMC v2 Advertisers: Advertiser ID: Failed", extra={'error': get_exception_message(ex)}, ) raise TuneReportingError( error_message=("TMC v2 Advertisers: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Advertisers: Advertiser ID:") self.logger.debug("TMC v2 Advertisers: Advertiser ID", extra={'response': json_response}) json_response_status_code = json_response['status_code'] http_status_successful = is_http_status_type( http_status_code=json_response_status_code, http_status_type=HttpStatusType.SUCCESSFUL) if not http_status_successful or not json_response['data']: raise TuneReportingError( error_message="TMC v2 Advertisers: Failed: {}".format( json_response_status_code), error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) if 'data' not in json_response or \ not json_response['data'] or \ len(json_response['data']) == 0: raise TuneReportingError( error_message="TMC v2 Advertisers: Advertiser ID: Failed", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) advertiser_data = json_response['data'][0] self.advertiser_id = advertiser_data.get('id', None) if self.advertiser_id is None: raise TuneReportingError( error_message="TMC v2 Advertisers: Advertiser ID: Failed", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) self.logger.info("TMC v2 Advertisers: Advertiser ID: {}".format( self.advertiser_id)) return True
def get_session_token(self, tmc_api_key, request_retry=None): """Generate session token is returned to provide access to service. Args: tmc_api_key: request_retry: Returns: """ self.logger.info("TMC v2 Session Authenticate: Get Token") self.api_key = tmc_api_key request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller="session/authenticate", action="api_key" ) request_params = \ { 'api_keys': tmc_api_key, "source": "multiverse" } try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v2_request_retry_func, request_retry_excps_func=None, request_label="TMC v2 Session Authenticate") except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Session Authenticate: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Session Authenticate: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v2 Session Authenticate: Failed: {}".format( get_exception_message(ex))) raise TuneReportingError( error_message=( "TMC v2 Session Authenticate: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Get Session Token: Validate", ) self.logger.debug( "TMC v2 Session Authenticate: Details:", extra={'response': json_response}, ) response_status_code = json_response.get('status_code', None) status_code_successful = is_http_status_type( http_status_code=response_status_code, http_status_type=HttpStatusType.SUCCESSFUL) response_errors = json_response.get('errors', None) response_error_message = "" if response_errors: if isinstance(response_errors, dict): error_message = response_errors.get('message', None) if error_message: if error_message.startswith("Invalid api key"): response_status_code = TuneReportingErrorCodes.UNAUTHORIZED response_error_message += error_message elif isinstance(response_errors, list): for response_error in response_errors: if isinstance(response_error, dict) \ and 'message' in response_error: error_message = response_error.get('message', None) if error_message: if error_message.startswith("Invalid api key"): response_status_code = TuneReportingErrorCodes.UNAUTHORIZED response_error_message += error_message error_code = response_status_code if not status_code_successful or not json_response['data']: raise TuneReportingError( error_message="TMC v2 Session Authenticate: Failed: {}".format( response_error_message), error_code=error_code, ) self.session_token = json_response['data'] self.logger.info("TMC v2 Session Authenticate", extra={'session_token': self.session_token}) self.logger.info("TMC v2 Session Authenticate: Finished") return True
def tmc_auth_v2_advertiser(tmc_api_key, logger_level=logging.NOTSET, logger_format=TuneLoggingFormat.JSON): """TMC Authentication :return: """ log.info("TMC v2 Advertiser: Authentication: Start") response_auth = None tune_v2_advertisers = \ TuneV2Advertisers( logger_level=logger_level, logger_format=logger_format ) try: try: if tune_v2_advertisers.get_advertiser_id( auth_type=TuneV2AuthenticationTypes.API_KEY, auth_value=tmc_api_key, request_retry=None): advertiser_id = tune_v2_advertisers.advertiser_id log.debug("TMC v2 Advertiser: {}".format(advertiser_id)) except TuneRequestBaseError as tmc_req_ex: pprint(tmc_req_ex.to_dict()) print(str(tmc_req_ex)) except TuneReportingError as tmc_rep_ex: pprint(tmc_rep_ex.to_dict()) print(str(tmc_rep_ex)) except Exception as ex: print_traceback(ex) print(get_exception_message(ex)) except AuthenticationError as auth_ex: log.error("TMC v2 Advertiser: Authentication: Failed", extra=auth_ex.to_dict()) raise TuneReportingAuthError( error_message="TMC v2 Advertiser: Authentication: Failed", errors=auth_ex.errors, error_request_curl=auth_ex.request_curl, error_code=auth_ex.remote_status, ) except Exception as ex: print_traceback(ex) error_code = TuneReportingErrorCodes.REP_ERR_SOFTWARE log.error('TMC v2 Advertiser: Authentication: Failed: Unexpected', extra={ 'exit_code': error_code, 'error_exception': base_class_name(ex), 'error_details': get_exception_message(ex) }) raise TuneReportingAuthError( error_message= "TMC v2 Advertiser: Authentication: Failed: Unexpected", errors=ex, error_code=error_code) if response_auth: log.debug("TMC v2 Advertiser: Authentication: Details", extra=response_auth.to_dict()) return response_auth
def _collect(self, auth_type_use, start_date, end_date, request_params=None, request_retry=None, request_action=TuneV2AdvertiserStatsActions.FIND): """Collect data: TUNE Advertiser Stats Actuals. Args: start_date: end_date: request_params: request_retry: request_action: Returns: """ self.logger.debug(("TuneV2AdvertiserStatsActuals: Collect: " "Action: '{}'").format(request_action)) dict_request_params = self._map_request_params(auth_type_use, start_date, end_date, request_params) self.logger.debug("TuneV2AdvertiserStatsActuals: Collect", extra={'build_params': dict_request_params}) try: if request_action == TuneV2AdvertiserStatsActions.FIND: self._find_v2(request_params=dict_request_params, request_retry=request_retry) elif request_action == TuneV2AdvertiserStatsActions.EXPORT: self._export_v2_download_csv( auth_type_use=auth_type_use, export_controller=self._CONTROLLER, export_action='export', export_status_controller=self._CONTROLLER, export_status_action='status', request_params=dict_request_params, request_retry=request_retry) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Collect: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Collect: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TuneV2AdvertiserStatsActuals: Collect: {}".format( get_exception_message(ex))) raise TuneReportingError( error_message=( "TuneV2AdvertiserStatsActuals: Collect: Failed: {}" ).format(get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
def _stream_v2( self, auth_type_use, start_date, end_date, request_params, request_retry=None, ): """Stream data: TUNE Advertiser Stats Actuals. Args: start_date: end_date: request_params: request_retry: Returns: """ self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export", extra={'request_params': request_params}) dict_request_params = self._map_request_params(auth_type_use, start_date, end_date, request_params) self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export", extra={'build_params': dict_request_params}) try: response = self._export_stream_v2( auth_type_use, export_controller=self._CONTROLLER, export_action='export', export_status_controller=self._CONTROLLER, export_status_action='status', request_params=dict_request_params, request_retry=request_retry) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Stream: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Stream: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TuneV2AdvertiserStatsActuals: {}".format( get_exception_message(ex))) raise TuneReportingError( error_message=( "TuneV2AdvertiserStatsActuals: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) return response
def _export_v3_job_to_queue(self, request_params, request_retry=None, request_label="TMC v3 Job On Queue"): """Gather data using action export.json. :param request_params: :param request_retry: :return: """ request_url = \ self.tune_mat_request_path( mat_api_version="v3", controller=self.controller, action="exports/{}".format( self.logs_advertisers_type ) ) self.logger.debug( msg=("TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'exports': " "Start Advertiser Report Actuals").format(self.logs_advertisers_type), extra={'start_date': request_params["start_date"], 'end_date': request_params["end_date"]} ) try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=request_retry, request_retry_http_status_codes=None, request_retry_func=self.tune_v3_request_retry_func, request_retry_excps_func=None, request_label=request_label ) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v3 Logs Advertisers Base: {}".format(get_exception_message(ex))) raise json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v3 Logs Advertisers: '{}': Action 'exports'".format(self.logs_advertisers_type) ) if ('handle' not in json_response or not json_response['handle']): raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'exports': " "Response missing 'handle': {}" ).format(self.logs_advertisers_type, str(json_response)), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE ) export_job = json_response['handle'] if not export_job: raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'exports': " "Response missing 'handle'" ).format(self.logs_advertisers_type), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE ) self.logger.info( "TuneV3LogsAdvertisersBase", extra={ 'advertiser_type': self.logs_advertisers_type, 'action': 'exports', 'start_date': request_params["start_date"], 'end_date': request_params["end_date"], 'job': export_job } ) return export_job
def _check_v3_job_status_on_queue(self, export_job, request_retry=None, request_label="TMC v3 Job Status On Queue"): """Status of Export Report. Args: export_job: request_retry: Returns: """ request_label = "v3 Logs Advertisers Check Export Status" request_url = \ self.tune_mat_request_path( mat_api_version="v3", controller=self.controller, action="exports/{}".format( export_job ) ) self.logger.info(( "TMC v3 Logs Advertisers Base: Check Export Status: " "Logs '{}': " "Action: 'exports status', " "Status of Export Report for " "Job Handle: '{}'" ).format(self.logs_advertisers_type, export_job)) tries = -1 # default: -1 (indefinite) delay = 10 jitter = 0 max_delay = 60 if request_retry: if 'delay' in request_retry: delay = request_retry['delay'] if 'jitter' in request_retry: jitter = request_retry['jitter'] if 'max_delay' in request_retry: max_delay = request_retry['max_delay'] request_params = {"session_token": self.session_token} self.logger.debug("TMC v3 Logs Advertisers Base: Check Export Status", extra={'request_url': request_url}) self.logger.debug( "TMC v3 Logs Advertisers Base: Check Export Status: Request Retry", extra={'tries': tries, 'delay': delay, 'jitter': jitter, 'max_delay': max_delay} ) self.logger.debug( "TMC v3 Logs Advertisers Base: Check Export Status: Request", extra={'request_params': safe_dict(request_params)} ) report_url = None _attempts = 1 export_percent_complete = 0 export_status_action = 'exports status' self.logger.warning( "TMC v3 Logs Advertisers Base: Check Export Status", extra={'job': export_job, 'attempt': _attempts, 'action': export_status_action} ) time.sleep(10) _tries, _delay = tries, delay while True: try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v3_request_retry_func, request_retry_excps_func=None, request_label=request_label ) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Check Export Status: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Check Export Status: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TMC v3 Logs Advertisers Base: Check Export Status: Failed", extra={'error': get_exception_message(ex)} ) raise http_status_successful = is_http_status_type( http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL ) if not http_status_successful: raise TuneReportingError( error_message="Failed to get export status on queue: {}".format(response.status_code), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST ) json_response = response.json() export_percent_complete = 0 if "percent_complete" in json_response: export_percent_complete = \ safe_int(json_response["percent_complete"]) self.logger.info( "TMC v3 Logs Advertisers Base: Check Job Export Status", extra={ 'job': export_job, 'response_status_code': json_response["status"], 'export_percent_complete': safe_int(export_percent_complete) } ) if (export_percent_complete == 100 and json_response["status"] == "complete" and json_response["url"]): report_url = json_response["url"] self.logger.info( "TMC v3 Logs Advertisers Base: Check Job Export Status: Completed", extra={ 'job': export_job, 'report_url': report_url, 'request_label': request_label, 'export_percent_complete': safe_int(export_percent_complete) } ) break if tries >= 0: _tries -= 1 if _tries == 0: self.logger.error(("TMC v3 Logs Advertisers Base: " "Check Job Export Status: Exhausted Retries"), extra={ 'attempt': _attempts, 'tries': _tries, 'action': export_status_action, 'request_label': request_label, 'export_percent_complete': export_percent_complete }) raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: " "Check Job Export Status: " "Exhausted Retries: " "Percent Completed: {}" ).format(export_percent_complete), error_code=TuneReportingErrorCodes.REP_ERR_JOB_STOPPED ) _attempts += 1 self.logger.warning( "TMC v3 Logs Advertisers Base: Check Export Status", extra={'attempt': _attempts, 'job': export_job, 'delay': _delay, 'action': 'exports status'} ) time.sleep(_delay) _delay += jitter _delay = min(_delay, max_delay) if export_percent_complete == 100 and not report_url: raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: Check Job Export Status: " "Download report URL: Undefined" ) ) self.logger.info( "TMC v3 Logs Advertisers Base: Check Job Export Status: Finished", extra={ 'attempt': _attempts, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label, 'export_percent_complete': export_percent_complete, 'job': export_job } ) return report_url
def main(tmc_api_key): TIMEZONE_COLLECT = "America/New_York" tune_v2_advertiser_stats_actuals = \ TuneV2AdvertiserStatsActuals( timezone=TIMEZONE_COLLECT, logger_level=logging.INFO, logger_format=TuneLoggingFormat.JSON ) dw_file_path = "data.{}".format(TuneV2AdvertiserStatsFormats.JSON) if os.path.exists(dw_file_path): os.remove(dw_file_path) tz = pytz.timezone(TIMEZONE_COLLECT) yesterday = datetime.now(tz).date() - timedelta(days=1) str_yesterday = str(yesterday) request_params = { 'format': TuneV2AdvertiserStatsFormats.CSV, 'fields': ("ad_clicks," "ad_clicks_unique," "ad_impressions," "ad_impressions_unique," "ad_network_id," "advertiser_id," "country.code," "date_hour," "events," "installs," "is_reengagement," "payouts," "publisher_id," "publisher_sub_ad.ref," "publisher_sub_adgroup.ref," "publisher_sub_campaign.ref," "publisher_sub_publisher.ref," "publisher_sub_site.ref," "site_id"), 'group': ("country_id," "is_reengagement," "publisher_id," "publisher_sub_ad_id," "publisher_sub_adgroup_id," "publisher_sub_campaign_id," "publisher_sub_publisher_id," "publisher_sub_site_id," "site_id"), 'timezone': "America/Los_Angeles" } try: tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key) response = tune_v2_advertiser_stats_actuals.stream( auth_value=tmc_api_key, auth_type=TuneV2AuthenticationTypes.API_KEY, auth_type_use=TuneV2AuthenticationTypes.API_KEY, start_date=str_yesterday, end_date=str_yesterday, request_params=request_params, request_retry={ 'delay': 15, 'timeout': 30, 'tries': 5 }) line_count = 0 csv_keys_list = None json_keys_dict = { "publisher_sub_campaign.ref": "sub_campaign_ref", "publisher_sub_ad.ref": "sub_ad_ref", "publisher_sub_adgroup.ref": "sub_adgroup_ref", "publisher_sub_publisher.ref": "sub_publisher_ref", "publisher_sub_site.ref": "sub_site_ref", "publisher_sub_placement.ref": "sub_placement_ref", "publisher_sub_campaign.name": "sub_campaign_name", "publisher_sub_ad.name": "sub_ad_name", "publisher_sub_adgroup.name": "sub_adgroup_name", "publisher_sub_publisher.name": "sub_publisher_name", "publisher_sub_site.name": "sub_site_name", "publisher_sub_placement.name": "sub_placement_name", "publisher_sub_campaign_id": "sub_campaign_id", "publisher_sub_ad_id": "sub_ad_id", "publisher_sub_adgroup_id": "sub_adgroup_id", "publisher_sub_publisher_id": "sub_publisher_id", "publisher_sub_site_id": "sub_site_id", "publisher_sub_placement_id": "publisher_sub_placement_id", "country.code": "country_code", "ad_impressions": "received_impressions_gross", "ad_impressions_unique": "received_impressions_unique", "ad_clicks": "received_clicks_gross", "ad_clicks_unique": "received_clicks_unique", "events": "received_engagements", "installs": "received_installs", "payouts": "cost" } json_types_dict = { "client_id": int, "partner_id": int, "vendor_id": int, "date": str, "hour": int, "timezone": str, "granularity": str, "site_ref_id": str, "site_ref_type": str, "partner_ref_id": int, "partner_ref_type": str, "partner_vendor_ref_id": int, "partner_vendor_ref_type": str, "sub_campaign_type": str, "sub_campaign_ref": str, "sub_ad_ref": str, "sub_adgroup_ref": str, "sub_publisher_ref": str, "sub_site_ref": str, "sub_placement_ref": str, "sub_campaign_name": str, "sub_ad_name": str, "sub_adgroup_name": str, "sub_publisher_name": str, "sub_site_name": str, "sub_placement_name": str, "sub_campaign_name": str, "sub_ad_name": str, "sub_adgroup_name": str, "sub_publisher_name": str, "sub_site_name": str, "sub_placement_name": str, "sub_campaign_id": int, "sub_ad_id": int, "sub_adgroup_id": int, "sub_publisher_id": int, "sub_site_id": int, "publisher_sub_placement_id": int, "country_code": str, "received_impressions_gross": int, "received_impressions_unique": int, "received_clicks_gross": int, "received_clicks_unique": int, "received_installs": int, "received_engagements": int, "received_conversions": int, "cost": float, "cost_currency": str, "site_id": int, "publisher_id": int, "advertiser_id": int, "ad_network_id": int, "ad_impressions": int, } client_id = 0 partner_id = 0 vendor_id = 0 timezone = "TBD" granularity = "TBD" config_extra = { "client_id": client_id, "partner_id": partner_id, "vendor_id": vendor_id, "timezone": timezone, "granularity": granularity, "cost_currency": "USD", "received_conversions": 0, "site_ref_type": "tmc", "partner_ref_type": "tmc", "partner_vendor_ref_type": "tmc" } with open(file=dw_file_path, mode='w') as dw_file_w: for bytes_line in response.iter_lines(chunk_size=4096): if bytes_line: # filter out keep-alive new chunks line_count += 1 str_line = bytes_line.decode("utf-8") if line_count == 1: csv_keys_list = str_line.split(',') for index, csv_key in enumerate(csv_keys_list): if csv_key in json_keys_dict: csv_keys_list[index] = json_keys_dict[csv_key] continue elif line_count > 2: dw_file_w.write('\n') csv_values_list = str_line.split(',') json__dict = {} is_reengagement = 0 received_installs = 0 received_engagements = 0 for csv_key, csv_value in zip(csv_keys_list, csv_values_list): csv_value_strip = csv_value.strip('"') if csv_key == "date_hour": parts_date_time = csv_value_strip.split(" ") rdate_yyyy_mm_dd = parts_date_time[0] parts_time = parts_date_time[1].split(":") rhour = safe_int(parts_time[0]) json__dict.update({"date": rdate_yyyy_mm_dd}) json__dict.update({"hour": rhour}) elif csv_key == "is_reengagement": is_reengagement = safe_int(csv_value_strip) elif csv_key == "received_installs": received_installs = safe_int(csv_value_strip) json__dict.update( {'received_installs': received_installs}) elif csv_key == "received_engagements": received_engagements = safe_int(csv_value_strip) json__dict.update( {'received_engagements': received_engagements}) else: if csv_key in json_types_dict: if json_types_dict[csv_key] == str: csv_value_typed = safe_str(csv_value_strip) elif json_types_dict[csv_key] == int: csv_value_typed = safe_int(csv_value_strip) elif json_types_dict[csv_key] == float: csv_value_typed = safe_float( csv_value_strip) else: csv_value_typed = safe_str(csv_value_strip) else: csv_value_typed = safe_str(csv_value_strip) json__dict.update({csv_key: csv_value_typed}) if is_reengagement == 1: engagements = received_engagements else: engagements = 0 if engagements > 0 and received_installs > 0: sub_campaign_type = "acquisition_engagement" elif received_installs > 0: sub_campaign_type = "acquisition" elif engagements > 0: sub_campaign_type = "engagement" else: sub_campaign_type = "" json__dict.update({'sub_campaign_type': sub_campaign_type}) json__dict.update(config_extra) json_str = json.dumps(json__dict) dw_file_w.write(json_str) dw_file_w.flush() statinfo = os.stat(dw_file_path) extra = { 'response_status_code': response.status_code, 'response_headers': response.headers, 'dw_file_path': dw_file_path, 'dw_file_size': convert_size(statinfo.st_size), 'line_count': line_count, 'csv_header_list': csv_keys_list } pprint(extra) except TuneRequestBaseError as tmc_req_ex: print_traceback(tmc_req_ex) pprint(tmc_req_ex.to_dict()) print(str(tmc_req_ex)) except TuneReportingError as tmc_rep_ex: pprint(tmc_rep_ex.to_dict()) print(str(tmc_rep_ex)) except Exception as ex: print_traceback(ex) print(get_exception_message(ex))
def main(tmc_api_key): tune_v2_advertiser_stats_actuals = \ TuneV2AdvertiserStatsActuals( logger_level=logging.INFO, logger_format=TuneLoggingFormat.JSON ) tz = pytz.timezone("America/New_York") yesterday = datetime.now(tz).date() - timedelta(days=1) str_yesterday = str(yesterday) try: auth_response = tune_v2_advertiser_stats_actuals.tmc_auth(tmc_api_key=tmc_api_key) assert auth_response tune_v2_advertiser_stats_actuals.collect( auth_value=tmc_api_key, auth_type=TuneV2AuthenticationTypes.API_KEY, auth_type_use=TuneV2AuthenticationTypes.API_KEY, start_date=str_yesterday, end_date=str_yesterday, request_params={ 'timezone': 'America/Los_Angeles', 'format': TuneV2AdvertiserStatsFormats.CSV, 'fields': ( "ad_clicks," "ad_clicks_unique," "ad_impressions," "ad_impressions_unique," "ad_network_id," "advertiser_id," "country.code," "date_hour," "events," "installs," "is_reengagement," "payouts," "publisher_id," "publisher_sub_ad.ref," "publisher_sub_adgroup.ref," "publisher_sub_campaign.ref," "publisher_sub_publisher.ref," "publisher_sub_site.ref," "site_id" ), 'group': ( "country_id," "is_reengagement," "publisher_id," "publisher_sub_ad_id," "publisher_sub_adgroup_id," "publisher_sub_campaign_id," "publisher_sub_publisher_id," "publisher_sub_site_id," "site_id" ), 'timezone': "America/Los_Angeles", 'limit': 5 }, request_action=TuneV2AdvertiserStatsActions.EXPORT, request_retry={'delay': 15, 'timeout': 30, 'tries': 10} ) except TuneRequestBaseError as tmc_req_ex: print_traceback(tmc_req_ex) pprint(tmc_req_ex.to_dict()) print(str(tmc_req_ex)) except TuneReportingError as tmc_rep_ex: pprint(tmc_rep_ex.to_dict()) print(str(tmc_rep_ex)) except Exception as ex: print_traceback(ex) print(get_exception_message(ex)) for row in list(tune_v2_advertiser_stats_actuals.generator): pprint(row)
def tmc_auth(self, tmc_api_key): """TMC Authentication""" if not tmc_api_key: raise ValueError("Parameter 'tmc_api_key' not defined.") self.logger.info("TMC Authentication: Start") # Attempt to authenticate. request_url = ('https://api.mobileapptracking.com/v2/advertiser/find') request_params = {'api_key': tmc_api_key} auth_request_curl = command_line_request_curl_get(request_url=request_url, request_params=request_params) try: auth_response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_label="TMC Authentication" ) except TuneRequestBaseError as tmc_ex: self.logger.error("TMC Authentication: Failed", extra=tmc_ex.to_dict()) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC Authentication: Failed: {}".format(get_exception_message(ex))) raise if auth_response.status_code != requests.codes.ok: raise TuneReportingError( error_message="Invalid request", error_request_curl=auth_request_curl, error_code=auth_response.status_code ) try: decoded_resp = auth_response.json() except Exception as ex: # No JSON response available. raise TuneReportingError( error_message='Invalid JSON response: {}'.format(auth_response.text), errors=ex, error_request_curl=auth_request_curl, error_code=TuneReportingErrorCodes.REP_ERR_AUTH_JSON_ERROR ) tmc_status_code = decoded_resp.get('status_code', None) tmc_errors = decoded_resp.get('errors', None) if tmc_errors: error_code = tmc_status_code errors = [] if isinstance(tmc_errors, list): error_list = tmc_errors error_list = error_list if error_list else [] for error in error_list: error_message = error.get('message', None) if error_message: errors.append(error_message) if error_message.startswith("Invalid api key"): error_code = TuneReportingErrorCodes.UNAUTHORIZED elif isinstance(tmc_errors, dict): error_message = tmc_errors.get('message', None) if error_message: errors.append(error_message) raise TuneReportingError( error_message="Error status: {}".format(tmc_errors), error_request_curl=auth_request_curl, error_code=error_code, ) return decoded_resp