def _read_stats_data(self, export_job_handle, report_url, report_format=None): if not export_job_handle: raise ValueError("Argument 'export_job_handle' not assigned.") if not report_url: raise ValueError("Argument 'report_url' not assigned.") self.logger.info(("TMC v3 Logs Advertisers Base: " "Start Advertiser Report Stats read: " "Job Handle: '{}'").format(export_job_handle)) if not report_format: path = urlparse(report_url).path ext = os.path.splitext(path)[1] report_format = ext[1:] self.logger.info(("TMC v3 Logs Advertisers Base: " "Reading URL '{}' with format '{}'").format(report_url, report_format)) # pylint: disable=redefined-variable-type if report_format == TuneV3LogsAdvertisersResponseFormats.JSON: report_reader = ReportReaderJSON(report_url) elif report_format == TuneV3LogsAdvertisersResponseFormats.CSV: report_reader = ReportReaderCSV(report_url) else: raise TuneReportingError(error_message="Unexpected report format: '{}'".format(report_format)) # pylint: enable=redefined-variable-type if not report_reader: raise TuneReportingError(error_message="Report reader not created for format {}".format(report_format)) report_reader.read() self.logger.info(("TMC v3 Logs Advertisers Base: " "Finished Advertiser Report Stats read")) return report_reader.generator
def _read_stats_data(self, export_job_id, report_url, report_format=None): """Read v2 Advertiser Stats Data Args: export_job_id: report_url: report_format: Returns: """ if not export_job_id: raise ValueError("Argument 'job_id' not assigned.") if not report_url: raise ValueError("Argument 'report_url' not assigned.") self.logger.info(msg=("TMC v2 Advertiser Stats: " "Start Advertiser Report Stats Read"), extra={'job_id': export_job_id}) if not report_format: path = urlparse(report_url).path ext = os.path.splitext(path)[1] report_format = ext[1:] self.logger.info("TMC v2 Advertiser Stats: Reading", extra={ 'report_url': report_url, 'report_format': report_format }) # pylint: disable=redefined-variable-type if report_format == TuneV2AdvertiserStatsFormats.JSON: report_reader = ReportReaderJSON(report_url) elif report_format == TuneV2AdvertiserStatsFormats.CSV: report_reader = ReportReaderCSV(report_url) else: raise TuneReportingError( error_message=( "Unexpected Report format: '{}'").format(report_format), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) # pylint: enable=redefined-variable-type if not report_reader: raise TuneReportingError( error_message=("Report reader not created for format {}" ).format(report_format), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) report_reader.read() self.logger.info(("TMC v2 Advertiser Stats: " "Finished Advertiser Report Stats read")) return report_reader.generator
def collect( self, auth_value, auth_type, auth_type_use, start_date, end_date, advertiser_id, request_params=None, request_retry=None, request_action=TuneV3LogsAdvertisersActions.FIND ): """Collect data: TUNE Advertiser Stats Actuals. Args: tmc_api_key: start_date: end_date: advertiser_id: request_params: request_retry: request_action: Returns: """ if not auth_value: raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'auth_value' not provided.") if not auth_type: raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'auth_type' not valid.") if not auth_type_use or \ not TuneV2AuthenticationTypes.validate(auth_type_use): raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'auth_type_use' not valid.") if not advertiser_id: raise ValueError("TMC v3 Logs Advertisers Base: Collect: Value 'advertiser_id' not valid.") if auth_type == TuneV2AuthenticationTypes.API_KEY: self.api_key = auth_value if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = tmc_auth_v2_session_token( tmc_api_key=self.api_key, logger_level=self.logger_level, logger_format=self.logger_format ) elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = auth_value else: raise ValueError(error_message="Invalid 'auth_type': '{}'".format(auth_type)) auth_value_use = None if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: if self.session_token is None: raise TuneReportingError( error_message="Value 'session_token' not defined.", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE ) auth_value_use = self.session_token elif auth_type_use == TuneV2AuthenticationTypes.API_KEY: if self.api_key is None: raise TuneReportingError( error_message="Value 'api_key' not defined.", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE, ) auth_value_use = self.api_key if not auth_value_use: raise TuneReportingError( error_message="Value 'auth_value_use' not defined.", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE, ) self.advertiser_id = advertiser_id self.controller = self._CONTROLLER.format(advertiser_id=self.advertiser_id) self._collect(auth_type_use, start_date, end_date, request_params, request_retry, request_action)
def _check_v2_job_status_on_queue( self, auth_type, auth_value, export_status_controller, export_status_action, export_job_id, request_retry=None, ): """Check Job Export Status Args: export_status_controller: export_status_action: export_job_id: request_retry: Returns: """ request_label = "TMC v2 Advertiser Stats: Check Export Status" v2_export_status_request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller=export_status_controller, action=export_status_action ) request_params = {auth_type: auth_value, "job_id": export_job_id} self.logger.info( ("TMC v2 Advertiser Stats: Check Job Status"), extra={ 'action': export_status_action, 'job_id': export_job_id, 'request_url': v2_export_status_request_url, 'request_params': safe_dict(request_params) }) tries = 60 # -1 (indefinite) delay = 10 jitter = 10 max_delay = 60 if request_retry is not None: if 'delay' in request_retry: delay = request_retry['delay'] if 'jitter' in request_retry: jitter = request_retry['jitter'] if 'max_delay' in request_retry: max_delay = request_retry['max_delay'] if 'tries' in request_retry: tries = request_retry['tries'] else: request_retry.update({'tries': 60}) else: request_retry = {'tries': 60, 'delay': 10, 'timeout': 60} self.logger.debug(msg=("TMC v2 Advertiser Stats: Check Job Status: " "Request Retry"), extra=request_retry) report_url = None _attempts = 1 export_percent_complete = 0 time.sleep(10) _tries, _delay = tries, delay while True: try: response = self.mv_request.request( request_method="GET", request_url=v2_export_status_request_url, request_params=request_params, request_label=request_label, request_retry_func=self.tune_v2_request_retry_func) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertiser Stats: Check Job Status: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertiser Stats: Check Job Status: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TMC v2 Advertiser Stats: Check Job Status: {}".format( get_exception_message(ex))) raise http_status_successful = is_http_status_type( http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL) if not http_status_successful: raise TuneReportingError( error_message=( "Failed to get export status on queue: {}").format( response.status_code), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST) if hasattr(response, 'url'): self.logger.info( "TMC v2 Advertiser Stats: Reporting API: Status URL", extra={'response_url': response.url}) json_response = response.json() if not json_response: request_status_successful = False elif 'status_code' not in json_response: request_status_successful = False else: status_code = json_response['status_code'] request_status_successful = is_http_status_type( http_status_code=status_code, http_status_type=HttpStatusType.SUCCESSFUL) errors = None if 'errors' in json_response: errors = json_response['errors'] if not request_status_successful: error_message = ( "TMC v2 Advertiser Stats: Check Job Status: GET '{}', Failed: {}, {}" ).format(v2_export_status_request_url, status_code, errors) if (status_code == TuneReportingError.EX_SRV_ERR_500_INTERNAL_SERVER): self.logger.error(error_message) elif (status_code == TuneReportingError.EX_SRV_ERR_503_SERVICE_UNAVAILABLE): self.logger.error(error_message) elif (status_code == TuneReportingError.EX_SRV_ERR_504_SERVICE_TIMEOUT): self.logger.error(error_message) continue elif (status_code == TuneReportingError.EX_CLT_ERR_408_REQUEST_TIMEOUT): self.logger.error( "GET '{}' request timeout, Retrying: {}".format( v2_export_status_request_url, status_code)) continue else: raise TuneReportingError(error_message=error_message, error_code=status_code) if tries >= 0 and _tries <= 1: if (status_code == HttpStatusCode.GATEWAY_TIMEOUT): raise TuneReportingError( error_message=error_message, error_code=TuneReportingErrorCodes.GATEWAY_TIMEOUT) elif (status_code == HttpStatusCode.REQUEST_TIMEOUT): raise TuneReportingError( error_message=error_message, error_code=TuneReportingErrorCodes.REQUEST_TIMEOUT) else: raise TuneReportingError(error_message=error_message, error_code=status_code) else: self.logger.warning(error_message) export_percent_complete = 0 if 'data' in json_response and json_response['data']: json_data = json_response['data'] if "percent_complete" in json_data: export_percent_complete = \ safe_int(json_data["percent_complete"]) self.logger.info(msg=("TMC v2 Advertiser Stats: " "Check Job Export Status: " "Response Success"), extra={ 'job_id': export_job_id, 'export_status': json_data["status"], 'export_percent_complete': safe_int(export_percent_complete), 'attempt': _attempts }) if (export_status_action == TuneV2AdvertiserStatsStatusAction.STATUS): if (export_percent_complete == 100 and json_data["status"] == "complete" and json_data["url"]): report_url = json_data["url"] self.logger.debug( ("TMC v2 Advertiser Stats: " "Check Job Export Status: Completed"), extra={ 'job_id': export_job_id, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label }) break elif (export_status_action == TuneV2AdvertiserStatsStatusAction.DOWNLOAD): if (export_percent_complete == 100 and json_data["status"] == "complete" and json_data["data"]["url"]): report_url = json_data["data"]["url"] self.logger.debug( ("TMC v2 Advertiser Stats: " "Check Job Export Status: Completed"), extra={ 'job_id': export_job_id, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label }) break else: self.logger.debug("TMC v2 Advertiser Stats: " "Check Job Export Status: " "No Data Available") if tries >= 0: _tries -= 1 if _tries == 0: self.logger.error( ("TMC v2 Advertiser Stats: " "Check Job Export Status: Exhausted Retries"), extra={ 'attempt': _attempts, 'tries': _tries, 'action': export_status_action, 'request_label': request_label, 'export_percent_complete': safe_int(export_percent_complete), 'job_id': export_job_id }) raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Check Job Export Status: " "Exhausted Retries: " "Percent Completed: {}").format( safe_int(export_percent_complete)), error_code=TuneReportingErrorCodes. REP_ERR_RETRY_EXHAUSTED) _attempts += 1 self.logger.info("TMC v2 Advertiser Stats: Check Job Status", extra={ 'attempt': _attempts, 'job_id': export_job_id, 'delay': _delay, 'action': export_status_action }) time.sleep(_delay) _delay += jitter _delay = min(_delay, max_delay) if export_percent_complete == 100 and not report_url: raise TuneReportingError( error_message=( "TMC v2 Advertiser Stats: Check Job Export Status: " "Download report URL: Undefined"), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) self.logger.info( "TMC v2 Advertiser Stats: Check Job Export Status: Finished", extra={ 'attempt': _attempts, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label, 'export_percent_complete': export_percent_complete, 'job_id': export_job_id }) return report_url
def _export_v2_job_to_queue( self, export_controller, export_action, request_params, request_retry=None, request_label="TMC v2 Job To Queue", ): """Export Report Request to Job Queue Args: export_controller: export_action: request_params: request_retry: Returns: Export Job ID """ request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller=export_controller, action=export_action ) self.logger.debug( ("TMC v2 Advertiser Stats: " "Start Advertiser Report Actuals"), extra={ 'action': export_action, 'start_date': request_params["start_date"], 'end_date': request_params["end_date"] }) try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_label=request_label, request_retry_func=self.tune_v2_request_retry_func) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertiser Stats: Request Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertiser Stats: Reporting Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error(get_exception_message(ex)) raise TuneReportingError( error_message=( "TMC v2 Advertiser Stats: Unexpected Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Advertiser Stats: Action '{}'".format( export_action)) if hasattr(response, 'url'): self.logger.info( "TMC v2 Advertiser Stats: Reporting API: Export URL", extra={'response_url': response.url}) if (not json_response or json_response['status_code'] != 200 or 'errors' in json_response): raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Action '{}': " "Failed to export stats: {}, {}").format( export_action, json_response['status_code'], json_response['errors']), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST) if ('data' not in json_response or not json_response['data']): raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Action '{}': " "Missing data").format(export_action), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) json_data = json_response['data'] export_job_id = None if export_action == TuneV2AdvertiserStatsExportActions.EXPORT: if ('job_id' not in json_data or not json_data['job_id']): raise TuneReportingError(error_message=( "TMC v2 Advertiser Stats: " "Action '{}': " "Response missing 'export_job_id': {}").format( export_action, str(json_data)), error_code=TuneReportingErrorCodes. REP_ERR_UNEXPECTED_VALUE) export_job_id = json_data['job_id'] elif export_action == TuneV2AdvertiserStatsExportActions.FIND_EXPORT_QUEUE: export_job_id = json_data if not export_job_id: raise TuneReportingError( error_message=( "TMC v2 Advertiser Stats: " "Action '{}': " "Response missing 'job_id'").format(export_action), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE) self.logger.info("TMC v2 Advertiser Stats: Reporting API: Job ID", extra={'job_id': export_job_id}) return export_job_id
def _process_export_stream_v2( self, auth_type_use, str_date_start, str_date_end, export_controller, export_action, export_status_controller, export_status_action, request_params, request_retry, request_label="TMC v2 Advertiser Stats Export Stream", ): """Process Export Job by Steaming Args: str_date_start: str_date_end: export_controller: export_action: export_status_controller: export_status_action: request_params: request_retry: Returns: """ auth_value = None if auth_type_use == TuneV2AuthenticationTypes.API_KEY: auth_value = self.api_key elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: auth_value = self.session_token str_date_start += " 00:00:00" str_date_end += " 23:59:59" request_params["start_date"] = \ str_date_start request_params["end_date"] = \ str_date_end self.logger.debug( "TMC v2 Advertiser Stats: Export Stream V2: Export Job to Queue", extra={ 'controller': export_controller, 'action': export_action, 'request_params': safe_dict(request_params), 'request_retry': safe_dict(request_retry) }) export_job_id = self._export_v2_job_to_queue( export_controller, export_action, request_params, request_retry, ) self.logger.debug( "TMC v2 Advertiser Stats: Export Stream V2: Check Job status on Queue", extra={ 'controller': export_status_controller, 'action': export_status_action, 'job_id': export_job_id }) export_report_url = self._check_v2_job_status_on_queue( auth_type_use, auth_value, export_status_controller, export_status_action, export_job_id, request_retry=request_retry) if not export_report_url: raise TuneReportingError( error_message="Export URL not defined", error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE, ) self.logger.info(("TMC v2 Advertiser Stats: " "Export Stream V2: Request Completed Job"), extra={ 'job_id': export_job_id, 'report_url': export_report_url }) response = self.mv_request.request(request_method="GET", request_url=export_report_url, stream=True, request_label=request_label) self.logger.info( "TMC v2 Advertiser Stats: Export Stream V2: Response Completed Job", extra={ 'response_status_code': response.status_code, 'response_headers': response.headers, 'job_id': export_job_id, 'report_url': export_report_url }) return response
def _process_export_download_csv_v2( self, tmp_directory, data, auth_type_use, str_date_start, str_date_end, export_controller, export_action, export_status_controller, export_status_action, request_params, request_retry, ): """Process Export Job by Reading Downloaded CSV Args: data: str_date_start: str_date_end: export_controller: export_action: export_status_controller: export_status_action: request_params: request_retry: Returns: """ auth_value = None if auth_type_use == TuneV2AuthenticationTypes.API_KEY: auth_value = self.api_key elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: auth_value = self.session_token str_date_start += " 00:00:00" str_date_end += " 23:59:59" request_params["start_date"] = \ str_date_start request_params["end_date"] = \ str_date_end export_job_id = self._export_v2_job_to_queue( export_controller, export_action, request_params, request_retry, ) export_report_url = self._check_v2_job_status_on_queue( auth_type_use, auth_value, export_status_controller, export_status_action, export_job_id, request_retry=request_retry, ) if not export_report_url: raise TuneReportingError( error_message="Export URL not defined", error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE, ) self.logger.info("TuneV2AdvertiserStatsBase", extra={ 'job_id': export_job_id, 'report_url': export_report_url }) tmp_csv_file_name = self._TUNE_ADVERTISING_STATS_TPL.format( job_id=export_job_id, format=TuneV2AdvertiserStatsFormats.CSV) tmp_csv_file_name = tmp_csv_file_name.replace('-', '_') job_row_count = 0 for row in self.mv_request.request_csv_download( tmp_csv_file_name=tmp_csv_file_name, tmp_directory=tmp_directory, request_method="GET", request_url=export_report_url, request_label="TMC v2 Advertiser Stats: Export: Download CSV"): # for row in list(generator_data_stats): if len(row) == 0: continue job_row_count += 1 data.append(row) if os.path.exists(tmp_csv_file_name): os.remove(tmp_csv_file_name) self.logger.debug( "TMC v2 Advertiser Stats: Process Export Job by Reading Downloaded CSV", extra={ 'action': 'export', 'start_date': str_date_start, 'end_date': str_date_end, 'job_id': export_job_id, 'row_count': job_row_count }) return (data, job_row_count)
def _map_request_params(self, auth_type_use, start_date, end_date, request_params=None): """Build Request Paramaters Args: start_date: end_date: request_params: Returns: """ auth_value = None if auth_type_use == TuneV2AuthenticationTypes.API_KEY: auth_value = self.api_key elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: auth_value = self.session_token dict_request_params = { auth_type_use: auth_value, "source": "multiverse", "response_timezone": self.timezone, "timestamp": "datehour", "group": ("advertiser_id," "country_id," "currency_code," "is_reengagement," "platform," "publisher_id," "publisher_sub_ad_id," "publisher_sub_adgroup_id," "publisher_sub_campaign_id," "publisher_sub_publisher_id," "publisher_sub_site_id," "purchase_validation_status," "site_id"), "fields": ("ad_clicks," "ad_clicks_unique," "ad_impressions," "ad_impressions_unique," "ad_network_id," "advertiser_id," "conversions," "country.code," "country.name," "currency_code," "date_hour," "events," "installs," "is_reengagement," "payouts," "publisher.name," "publisher_id," "publisher_sub_ad.ref," "publisher_sub_adgroup.ref," "publisher_sub_campaign.ref," "publisher_sub_publisher.ref," "publisher_sub_site.ref," "site.mobile_app_type," "site.package_name," "site.store_app_id," "site_id"), "filter": "({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA), "start_date": start_date, "end_date": end_date, "debug": 0 } if request_params: self.logger.debug( "TuneV2AdvertiserStatsActuals: Request", extra={'request_params': safe_dict(request_params)}) if "fields" in request_params and \ request_params["fields"]: dict_request_params["fields"] = \ request_params["fields"] if "group" in request_params and \ request_params["group"]: dict_request_params["group"] = \ request_params["group"] if "timestamp" in request_params and \ request_params["timestamp"]: dict_request_params["timestamp"] = \ request_params["timestamp"] if "filter" in request_params and \ request_params["filter"]: dict_request_params["filter"] = "({} AND {})".format( request_params["filter"], self._FILTER_NOT_DEBUG_NOR_TEST_DATA) if "format" in request_params: dict_request_params["format"] = \ request_params["format"] if "offset" in request_params: dict_request_params["offset"] = \ int(request_params["offset"]) if "page" in request_params: dict_request_params["page"] = \ int(request_params["page"]) if "limit" in request_params: dict_request_params["limit"] = \ int(request_params["limit"]) if "debug" in request_params: dict_request_params["debug"] = \ int(request_params["debug"]) response_timezone = None if "timezone" in request_params: response_timezone = request_params["timezone"] if "response_timezone" in request_params: response_timezone = request_params["response_timezone"] if response_timezone: if not validate_tz_name(response_timezone): return TuneReportingError( error_message="Invalid Timezone: {}".format( response_timezone)) self.timezone = response_timezone dict_request_params["response_timezone"] = \ self.timezone self.logger.debug(("TuneV2AdvertiserStatsActuals: " "Timezone: {}").format(self.timezone)) return dict_request_params
def tmc_auth(self, tmc_api_key): """TMC Authentication""" if not tmc_api_key: raise ValueError("Parameter 'tmc_api_key' not defined.") self.logger.info("TMC Authentication: Start") # Attempt to authenticate. request_url = ('https://api.mobileapptracking.com/v2/advertiser/find') request_params = {'api_key': tmc_api_key} auth_request_curl = command_line_request_curl_get(request_url=request_url, request_params=request_params) try: auth_response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=None, request_retry_excps_func=None, request_label="TMC Authentication" ) except TuneRequestBaseError as tmc_ex: self.logger.error("TMC Authentication: Failed", extra=tmc_ex.to_dict()) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC Authentication: Failed: {}".format(get_exception_message(ex))) raise if auth_response.status_code != requests.codes.ok: raise TuneReportingError( error_message="Invalid request", error_request_curl=auth_request_curl, error_code=auth_response.status_code ) try: decoded_resp = auth_response.json() except Exception as ex: # No JSON response available. raise TuneReportingError( error_message='Invalid JSON response: {}'.format(auth_response.text), errors=ex, error_request_curl=auth_request_curl, error_code=TuneReportingErrorCodes.REP_ERR_AUTH_JSON_ERROR ) tmc_status_code = decoded_resp.get('status_code', None) tmc_errors = decoded_resp.get('errors', None) if tmc_errors: error_code = tmc_status_code errors = [] if isinstance(tmc_errors, list): error_list = tmc_errors error_list = error_list if error_list else [] for error in error_list: error_message = error.get('message', None) if error_message: errors.append(error_message) if error_message.startswith("Invalid api key"): error_code = TuneReportingErrorCodes.UNAUTHORIZED elif isinstance(tmc_errors, dict): error_message = tmc_errors.get('message', None) if error_message: errors.append(error_message) raise TuneReportingError( error_message="Error status: {}".format(tmc_errors), error_request_curl=auth_request_curl, error_code=error_code, ) return decoded_resp
def collect(self, auth_value, auth_type, auth_type_use, site_ids=None, request_params=None, site_status=TuneV2AdvertiserSiteStatus.ACTIVE): """Collect TMC v2 Advertiser Sites Args: tmc_api_key: site_ids: request_params: request_retry: site_status: Returns: """ if not auth_value: raise ValueError( "TuneV2AdvertiserSites: Collect: Value 'auth_value' not provided." ) if not auth_type: raise ValueError( "TuneV2AdvertiserSites: Collect: Value 'auth_type' not valid.") if not auth_type_use or \ not TuneV2AuthenticationTypes.validate(auth_type_use): raise ValueError( "TMC v3 Logs Advertisers Base: Collect: Value 'auth_type_use' not valid." ) _request_params = {"source": "multiverse"} if auth_type == TuneV2AuthenticationTypes.API_KEY: self.api_key = auth_value if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = tmc_auth_v2_session_token( tmc_api_key=self.api_key, logger_level=self.logger_level, logger_format=self.logger_format) _request_params.update({ "session_token": self.session_token, }) else: _request_params.update({ "api_key": self.api_key, }) elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = auth_value _request_params.update({ "session_token": self.session_token, }) else: raise ValueError("Invalid 'auth_type': '{}'".format(auth_type)) request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller="advertiser/sites", action="find" ) self.logger.info("Start Advertiser Sites find") filter_status = None if site_status == TuneV2AdvertiserSiteStatus.ACTIVE: filter_status = "(status = \"active\")" elif site_status == TuneV2AdvertiserSiteStatus.NOT_ACTIVE: filter_status = "(status != \"active\")" filter_sites = None if site_ids and len(site_ids) > 0: filter_sites = "(id IN (" + ",".join(site_ids) + "))" if filter_status and filter_sites: _request_params["filter"] = "{} AND {}".format( filter_status, filter_sites) elif filter_status: _request_params["filter"] = filter_status elif filter_sites: _request_params["filter"] = filter_sites if request_params: if "filter" in request_params and request_params["filter"]: if "filter" in _request_params and _request_params["filter"]: _request_params["filter"] = "({} AND {})".format( _request_params["filter"], request_params["filter"]) else: _request_params["filter"] = "({})".format( request_params["filter"]) _request_params.update({'limit': request_params.get('limit', 0)}) else: _request_params.update({'limit': 0}) response = None try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=_request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v2_request_retry_func, request_retry_excps_func=None, request_label="{}.{}".format(self.__class__.__name__, sys._getframe().f_code.co_name)) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TuneV2AdvertiserSites: Collect: Failed", extra=tmc_req_ex.to_dict(), ) yield (None, tmc_req_ex) except TuneReportingError as tmc_rep_ex: self.logger.error( "TuneV2AdvertiserSites: Collect: Failed", extra=tmc_rep_ex.to_dict(), ) yield (None, tmc_rep_ex) except Exception as ex: print_traceback(ex) self.logger.error(get_exception_message(ex)) yield (None, ex) if response: json_response = response.json() if not json_response or \ json_response['status_code'] != 200 or \ 'errors' in json_response: raise TuneReportingError( error_message="Failed to get advertiser sites: {}, {}". format( json_response['status_code'], json_response['errors'].get('message', None), ), error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) if ('data' not in json_response or not json_response['data']): raise TuneReportingError( error_message="Missing 'data': {}".format( str(json_response)), error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) data = json_response['data'] else: data = None if not data or len(data) == 0: yield ([], None) else: for (i, item) in enumerate(data): yield (item, None)
def tune_v2_request_retry_func(self, response): """Request Retry Function Args: response: Returns: Boolean """ try: response_json = response.json() except Exception as ex: # No JSON response available. raise TuneReportingError( error_message='Invalid JSON response: {}'.format(response.text), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_JSON_DECODING ) self.logger.debug("TMC API V2: Check for Retry: Start", extra=response_json) tune_v2_status_code = None tune_v2_errors_messages = "" if 'status_code' in response_json: tune_v2_status_code = response_json['status_code'] if 'errors' in response_json: tune_v2_errors = response_json['errors'] if isinstance(tune_v2_errors, dict) \ and 'message' in tune_v2_errors: tune_v2_errors_messages += tune_v2_errors['message'] elif isinstance(tune_v2_errors, list): for tune_v2_error in tune_v2_errors: if isinstance(tune_v2_error, dict) \ and 'message' in tune_v2_error: tune_v2_errors_messages += tune_v2_error['message'] tune_v2_status_type = get_http_status_type(tune_v2_status_code) response_extra = { 'status_code': tune_v2_status_code, 'status_type': tune_v2_status_type, } if tune_v2_errors_messages: response_extra.update({'error_messages': safe_str(tune_v2_errors_messages)}) self.logger.debug("TMC API Base: Check for Retry: Response", extra=response_extra) if tune_v2_status_code == 200: self.logger.debug("TMC API Base: Check for Retry: Success", extra=response_extra) return False if tune_v2_status_code in [401, 403]: self.logger.error("TMC API: Request: Error", extra={'status_code': tune_v2_status_code}) raise TuneRequestError( error_message=tune_v2_errors_messages, error_code=tune_v2_status_code, ) if tune_v2_status_code in [404, 500]: if "Api key was not found." in tune_v2_errors_messages: self.logger.error("TMC API: Request: Error", extra={'tune_v2_status_code': tune_v2_status_code}) raise TuneRequestError( error_message=tune_v2_errors_messages, error_code=tune_v2_status_code, ) self.logger.warning("TMC API Base: Check for Retry: Retry Candidate", extra=response_extra) return True self.logger.warning("TMC API Base: Check for Retry: No Retry", extra=response_extra) return False
def get_advertiser_id(self, auth_type, auth_value, request_retry=None): """Get Advertiser ID Args: auth_type: auth_value: request_retry: Returns: """ if not auth_type: raise ValueError( "TMC v2 Advertisers: Get Advertiser ID: Value 'auth_type' not provided." ) if not auth_value: raise ValueError( "TMC v2 Advertisers: Get Advertiser ID: Value 'auth_value' not provided." ) request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller="advertiser", action="find" ) request_params = {auth_type: auth_value} self.logger.info("TMC v2 Advertisers: Advertiser ID") try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v2_request_retry_func, request_retry_excps_func=None, request_label="TMC v2 Advertisers") except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertisers: Advertiser ID: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertisers: Advertiser ID: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TMC v2 Advertisers: Advertiser ID: Failed", extra={'error': get_exception_message(ex)}, ) raise TuneReportingError( error_message=("TMC v2 Advertisers: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Advertisers: Advertiser ID:") self.logger.debug("TMC v2 Advertisers: Advertiser ID", extra={'response': json_response}) json_response_status_code = json_response['status_code'] http_status_successful = is_http_status_type( http_status_code=json_response_status_code, http_status_type=HttpStatusType.SUCCESSFUL) if not http_status_successful or not json_response['data']: raise TuneReportingError( error_message="TMC v2 Advertisers: Failed: {}".format( json_response_status_code), error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) if 'data' not in json_response or \ not json_response['data'] or \ len(json_response['data']) == 0: raise TuneReportingError( error_message="TMC v2 Advertisers: Advertiser ID: Failed", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) advertiser_data = json_response['data'][0] self.advertiser_id = advertiser_data.get('id', None) if self.advertiser_id is None: raise TuneReportingError( error_message="TMC v2 Advertisers: Advertiser ID: Failed", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) self.logger.info("TMC v2 Advertisers: Advertiser ID: {}".format( self.advertiser_id)) return True
def get_session_token(self, tmc_api_key, request_retry=None): """Generate session token is returned to provide access to service. Args: tmc_api_key: request_retry: Returns: """ self.logger.info("TMC v2 Session Authenticate: Get Token") self.api_key = tmc_api_key request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller="session/authenticate", action="api_key" ) request_params = \ { 'api_keys': tmc_api_key, "source": "multiverse" } try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v2_request_retry_func, request_retry_excps_func=None, request_label="TMC v2 Session Authenticate") except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Session Authenticate: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Session Authenticate: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v2 Session Authenticate: Failed: {}".format( get_exception_message(ex))) raise TuneReportingError( error_message=( "TMC v2 Session Authenticate: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Get Session Token: Validate", ) self.logger.debug( "TMC v2 Session Authenticate: Details:", extra={'response': json_response}, ) response_status_code = json_response.get('status_code', None) status_code_successful = is_http_status_type( http_status_code=response_status_code, http_status_type=HttpStatusType.SUCCESSFUL) response_errors = json_response.get('errors', None) response_error_message = "" if response_errors: if isinstance(response_errors, dict): error_message = response_errors.get('message', None) if error_message: if error_message.startswith("Invalid api key"): response_status_code = TuneReportingErrorCodes.UNAUTHORIZED response_error_message += error_message elif isinstance(response_errors, list): for response_error in response_errors: if isinstance(response_error, dict) \ and 'message' in response_error: error_message = response_error.get('message', None) if error_message: if error_message.startswith("Invalid api key"): response_status_code = TuneReportingErrorCodes.UNAUTHORIZED response_error_message += error_message error_code = response_status_code if not status_code_successful or not json_response['data']: raise TuneReportingError( error_message="TMC v2 Session Authenticate: Failed: {}".format( response_error_message), error_code=error_code, ) self.session_token = json_response['data'] self.logger.info("TMC v2 Session Authenticate", extra={'session_token': self.session_token}) self.logger.info("TMC v2 Session Authenticate: Finished") return True
def _collect(self, auth_type_use, start_date, end_date, request_params=None, request_retry=None, request_action=TuneV2AdvertiserStatsActions.FIND): """Collect data: TUNE Advertiser Stats Actuals. Args: start_date: end_date: request_params: request_retry: request_action: Returns: """ self.logger.debug(("TuneV2AdvertiserStatsActuals: Collect: " "Action: '{}'").format(request_action)) dict_request_params = self._map_request_params(auth_type_use, start_date, end_date, request_params) self.logger.debug("TuneV2AdvertiserStatsActuals: Collect", extra={'build_params': dict_request_params}) try: if request_action == TuneV2AdvertiserStatsActions.FIND: self._find_v2(request_params=dict_request_params, request_retry=request_retry) elif request_action == TuneV2AdvertiserStatsActions.EXPORT: self._export_v2_download_csv( auth_type_use=auth_type_use, export_controller=self._CONTROLLER, export_action='export', export_status_controller=self._CONTROLLER, export_status_action='status', request_params=dict_request_params, request_retry=request_retry) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Collect: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Collect: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TuneV2AdvertiserStatsActuals: Collect: {}".format( get_exception_message(ex))) raise TuneReportingError( error_message=( "TuneV2AdvertiserStatsActuals: Collect: Failed: {}" ).format(get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE)
def _export_v3_job_to_queue(self, request_params, request_retry=None, request_label="TMC v3 Job On Queue"): """Gather data using action export.json. :param request_params: :param request_retry: :return: """ request_url = \ self.tune_mat_request_path( mat_api_version="v3", controller=self.controller, action="exports/{}".format( self.logs_advertisers_type ) ) self.logger.debug( msg=("TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'exports': " "Start Advertiser Report Actuals").format(self.logs_advertisers_type), extra={'start_date': request_params["start_date"], 'end_date': request_params["end_date"]} ) try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=request_retry, request_retry_http_status_codes=None, request_retry_func=self.tune_v3_request_retry_func, request_retry_excps_func=None, request_label=request_label ) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v3 Logs Advertisers Base: {}".format(get_exception_message(ex))) raise json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v3 Logs Advertisers: '{}': Action 'exports'".format(self.logs_advertisers_type) ) if ('handle' not in json_response or not json_response['handle']): raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'exports': " "Response missing 'handle': {}" ).format(self.logs_advertisers_type, str(json_response)), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE ) export_job = json_response['handle'] if not export_job: raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: " "Logs '{}': " "Action 'exports': " "Response missing 'handle'" ).format(self.logs_advertisers_type), error_code=TuneReportingErrorCodes.REP_ERR_UNEXPECTED_VALUE ) self.logger.info( "TuneV3LogsAdvertisersBase", extra={ 'advertiser_type': self.logs_advertisers_type, 'action': 'exports', 'start_date': request_params["start_date"], 'end_date': request_params["end_date"], 'job': export_job } ) return export_job
def stream( self, auth_value, auth_type, auth_type_use, start_date, end_date, request_params, request_retry=None, ): """Stream data: TUNE Advertiser Stats Base Args: auth_value: auth_type: start_date: end_date: request_params: request_retry: Returns: """ self.logger.info("TMC v2 Advertiser Stats: Stream") if not auth_value: raise ValueError( error_message= "TMC v2 Advertiser Stats: Stream: Value 'auth_value' not provided." ) if not auth_type or \ not TuneV2AuthenticationTypes.validate(auth_type): raise ValueError( error_message= "TMC v2 Advertiser Stats: Stream: Value 'auth_type' not valid." ) if not auth_type_use or \ not TuneV2AuthenticationTypes.validate(auth_type_use): raise ValueError( error_message= "TMC v2 Advertiser Stats: Stream: Value 'auth_type_use' not valid." ) if not request_params: raise ValueError( error_message="Value 'request_params' not provided.") if auth_type == TuneV2AuthenticationTypes.API_KEY: self.api_key = auth_value if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = tmc_auth_v2_session_token( tmc_api_key=self.api_key, logger_level=self.logger_level, logger_format=self.logger_format) elif auth_type == TuneV2AuthenticationTypes.SESSION_TOKEN: self.session_token = auth_value else: raise ValueError( error_message="Invalid 'auth_type': '{}'".format(auth_type)) auth_value_use = None if auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: if self.session_token is None: raise TuneReportingError( error_message="Value 'session_token' not defined.", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) auth_value_use = self.session_token elif auth_type_use == TuneV2AuthenticationTypes.API_KEY: if self.api_key is None: raise TuneReportingError( error_message="Value 'api_key' not defined.", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE, ) auth_value_use = self.api_key if not auth_value_use: raise TuneReportingError( error_message="Value 'auth_value_use' not defined.", error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) return self._stream_v2(auth_type_use=auth_type_use, start_date=start_date, end_date=end_date, request_params=request_params, request_retry=request_retry)
def _check_v3_job_status_on_queue(self, export_job, request_retry=None, request_label="TMC v3 Job Status On Queue"): """Status of Export Report. Args: export_job: request_retry: Returns: """ request_label = "v3 Logs Advertisers Check Export Status" request_url = \ self.tune_mat_request_path( mat_api_version="v3", controller=self.controller, action="exports/{}".format( export_job ) ) self.logger.info(( "TMC v3 Logs Advertisers Base: Check Export Status: " "Logs '{}': " "Action: 'exports status', " "Status of Export Report for " "Job Handle: '{}'" ).format(self.logs_advertisers_type, export_job)) tries = -1 # default: -1 (indefinite) delay = 10 jitter = 0 max_delay = 60 if request_retry: if 'delay' in request_retry: delay = request_retry['delay'] if 'jitter' in request_retry: jitter = request_retry['jitter'] if 'max_delay' in request_retry: max_delay = request_retry['max_delay'] request_params = {"session_token": self.session_token} self.logger.debug("TMC v3 Logs Advertisers Base: Check Export Status", extra={'request_url': request_url}) self.logger.debug( "TMC v3 Logs Advertisers Base: Check Export Status: Request Retry", extra={'tries': tries, 'delay': delay, 'jitter': jitter, 'max_delay': max_delay} ) self.logger.debug( "TMC v3 Logs Advertisers Base: Check Export Status: Request", extra={'request_params': safe_dict(request_params)} ) report_url = None _attempts = 1 export_percent_complete = 0 export_status_action = 'exports status' self.logger.warning( "TMC v3 Logs Advertisers Base: Check Export Status", extra={'job': export_job, 'attempt': _attempts, 'action': export_status_action} ) time.sleep(10) _tries, _delay = tries, delay while True: try: response = self.mv_request.request( request_method="GET", request_url=request_url, request_params=request_params, request_retry=None, request_retry_http_status_codes=None, request_retry_func=self.tune_v3_request_retry_func, request_retry_excps_func=None, request_label=request_label ) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Check Export Status: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Base: Check Export Status: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error( "TMC v3 Logs Advertisers Base: Check Export Status: Failed", extra={'error': get_exception_message(ex)} ) raise http_status_successful = is_http_status_type( http_status_code=response.status_code, http_status_type=HttpStatusType.SUCCESSFUL ) if not http_status_successful: raise TuneReportingError( error_message="Failed to get export status on queue: {}".format(response.status_code), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST ) json_response = response.json() export_percent_complete = 0 if "percent_complete" in json_response: export_percent_complete = \ safe_int(json_response["percent_complete"]) self.logger.info( "TMC v3 Logs Advertisers Base: Check Job Export Status", extra={ 'job': export_job, 'response_status_code': json_response["status"], 'export_percent_complete': safe_int(export_percent_complete) } ) if (export_percent_complete == 100 and json_response["status"] == "complete" and json_response["url"]): report_url = json_response["url"] self.logger.info( "TMC v3 Logs Advertisers Base: Check Job Export Status: Completed", extra={ 'job': export_job, 'report_url': report_url, 'request_label': request_label, 'export_percent_complete': safe_int(export_percent_complete) } ) break if tries >= 0: _tries -= 1 if _tries == 0: self.logger.error(("TMC v3 Logs Advertisers Base: " "Check Job Export Status: Exhausted Retries"), extra={ 'attempt': _attempts, 'tries': _tries, 'action': export_status_action, 'request_label': request_label, 'export_percent_complete': export_percent_complete }) raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: " "Check Job Export Status: " "Exhausted Retries: " "Percent Completed: {}" ).format(export_percent_complete), error_code=TuneReportingErrorCodes.REP_ERR_JOB_STOPPED ) _attempts += 1 self.logger.warning( "TMC v3 Logs Advertisers Base: Check Export Status", extra={'attempt': _attempts, 'job': export_job, 'delay': _delay, 'action': 'exports status'} ) time.sleep(_delay) _delay += jitter _delay = min(_delay, max_delay) if export_percent_complete == 100 and not report_url: raise TuneReportingError( error_message=( "TMC v3 Logs Advertisers Base: Check Job Export Status: " "Download report URL: Undefined" ) ) self.logger.info( "TMC v3 Logs Advertisers Base: Check Job Export Status: Finished", extra={ 'attempt': _attempts, 'action': export_status_action, 'report_url': report_url, 'request_label': request_label, 'export_percent_complete': export_percent_complete, 'job': export_job } ) return report_url
def _find_v2( self, request_params, request_retry=None, request_label="TMC v2 Advertiser Stats Find", ): """Gather data using action find.json. """ self.logger.debug("TuneV2AdvertiserStatsBase", extra={'action': 'find'}) if "start_date" not in request_params: raise ValueError( "Missing attribute 'start_date' in parameter 'request_params'") request_start_date = request_params["start_date"] if "end_date" not in request_params: raise ValueError( "Missing attribute 'end_date' in parameter 'request_params'") request_end_date = request_params["end_date"] datetime_start = dt.datetime.strptime(request_start_date, '%Y-%m-%d') datetime_end = dt.datetime.strptime(request_end_date, '%Y-%m-%d') str_date_start = str(datetime_start.date()) str_date_end = str(datetime_end.date()) self.logger.debug( ("TuneV2AdvertiserStatsBase"), extra={ 'action': 'find', 'start_date': str_date_start, 'end_date': str_date_end }) if not request_retry: request_retry = {'timeout': 60} request_url = \ self.tune_mat_request_path( mat_api_version="v2", controller=self.controller, action="find" ) if 'format' not in request_params: request_params['format'] = TuneV2AdvertiserStatsFormats.JSON request_params["start_date"] += " 00:00:00" request_params["end_date"] += " 23:59:59" try: response = self.mv_request.request(request_method="GET", request_url=request_url, request_params=request_params, request_label=request_label) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v2 Advertiser Stats: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v2 Advertiser Stats: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v2 Advertiser Stats: {}".format( get_exception_message(ex))) raise json_response = validate_json_response( response, request_curl=self.mv_request.built_request_curl, request_label="TMC v2 Advertiser Stats: Action 'find'") if json_response['status_code'] != 200: raise TuneReportingError( error_message=("TMC v2 Advertiser Stats: " "Action 'find': " "Failed to find stats: {}, {}").format( json_response['status_code'], json.dumps(json_response), ), error_code=TuneReportingErrorCodes.REP_ERR_REQUEST) data = json_response['data'] self.data = data
def _collect( self, auth_type_use, start_date, end_date, request_params=None, request_retry=None, request_action=TuneV3LogsAdvertisersActions.FIND ): """Collect data: TUNE Advertiser Stats Clicks. Args: start_date: end_date: request_params: request_retry: request_action: Returns: """ auth_value = None if auth_type_use == TuneV2AuthenticationTypes.API_KEY: auth_value = self.api_key elif auth_type_use == TuneV2AuthenticationTypes.SESSION_TOKEN: auth_value = self.session_token dict_request_params = { auth_type_use: auth_value, "timezone": self.timezone, "fields": ( "created," "ad_network_id," "campaign.id," "campaign.name," "publisher.id," "publisher.name," "publisher_ref_id," "publisher_sub_site.id," "publisher_sub_site.ref," "publisher_sub_site.name," "publisher_sub_campaign.id," "publisher_sub_campaign.ref," "publisher_sub_campaign.name," "request_url," "site.id," "site.mobile_app_type," "site.name," "site.package_name," "site.store_app_id" ), "filter": "({})".format(self._FILTER_NOT_DEBUG_NOR_TEST_DATA), "start_date": start_date, "end_date": end_date, "debug": 0 } if request_params: if "fields" in request_params and request_params["fields"]: dict_request_params["fields"] = \ request_params["fields"] if "group" in request_params and request_params["group"]: dict_request_params["group"] = \ request_params["group"] if "timestamp" in request_params and request_params["timestamp"]: dict_request_params["timestamp"] = \ request_params["timestamp"] if "filter" in request_params and request_params["filter"]: dict_request_params["filter"] = "({} AND {})".format( request_params["filter"], self._FILTER_NOT_DEBUG_NOR_TEST_DATA ) if "limit" in request_params: dict_request_params["limit"] = \ int(request_params["limit"]) if "debug" in request_params: dict_request_params["debug"] = \ int(request_params["debug"]) timezone = None if "timezone" in request_params: timezone = request_params["timezone"] if timezone: if not validate_tz_name(timezone): return TuneReportingError(error_message="Invalid Timezone: {}".format(timezone)) self.timezone = timezone dict_request_params["timezone"] = \ self.timezone self.logger.debug(("TMC v3 Logs Advertisers Clicks: " "Action '{}', Params: {}").format(request_action, str(dict_request_params))) self.logger.debug(("TMC v3 Logs Advertisers Clicks: " "Timezone: {}").format(self.timezone)) try: if request_action == TuneV3LogsAdvertisersActions.FIND: request_params["sorts"] = "created desc" self._find_v3(request_params=dict_request_params, request_retry=request_retry) elif request_action == TuneV3LogsAdvertisersActions.EXPORT: self._export_v3_download_csv(request_params=dict_request_params, request_retry=request_retry) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TMC v3 Logs Advertisers Clicks: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TMC v3 Logs Advertisers Clicks: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TMC v3 Logs Advertisers Clicks: Failed: {}".format(get_exception_message(ex))) raise TuneReportingError( error_message=("TMC v3 Logs Advertisers Clicks: Failed: {}").format(get_exception_message(ex)), errors=ex )
def _stream_v2( self, auth_type_use, start_date, end_date, request_params, request_retry=None, ): """Stream data: TUNE Advertiser Stats Actuals. Args: start_date: end_date: request_params: request_retry: Returns: """ self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export", extra={'request_params': request_params}) dict_request_params = self._map_request_params(auth_type_use, start_date, end_date, request_params) self.logger.debug("TuneV2AdvertiserStatsActuals: Stream: Export", extra={'build_params': dict_request_params}) try: response = self._export_stream_v2( auth_type_use, export_controller=self._CONTROLLER, export_action='export', export_status_controller=self._CONTROLLER, export_status_action='status', request_params=dict_request_params, request_retry=request_retry) except TuneRequestBaseError as tmc_req_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Stream: Failed", extra=tmc_req_ex.to_dict(), ) raise except TuneReportingError as tmc_rep_ex: self.logger.error( "TuneV2AdvertiserStatsActuals: Stream: Failed", extra=tmc_rep_ex.to_dict(), ) raise except Exception as ex: print_traceback(ex) self.logger.error("TuneV2AdvertiserStatsActuals: {}".format( get_exception_message(ex))) raise TuneReportingError( error_message=( "TuneV2AdvertiserStatsActuals: Failed: {}").format( get_exception_message(ex)), errors=ex, error_code=TuneReportingErrorCodes.REP_ERR_SOFTWARE) return response