def __call__(self, *args, **kwargs) -> Iterator[ExportContext]: import pygsheets from google.auth.exceptions import ( RefreshError, OAuthError, GoogleAuthError, UserAccessTokenError, ) from googleapiclient import errors from pygsheets import Worksheet self.logger.info("Exportation data") self.export: "GoogleSheetsExportPolicy" = self.model_templating( *args, model=self.notebook.export) api = pygsheets.authorize(**self.export.credentials.dict()) try: sheet = api.open_by_key(self.export.sheet_id) page: Worksheet = sheet.worksheet(property="id", value=self.export.page_id) data = page.get_values( start=self.export.start, end=self.export.end, returnas="matrix", majdim="COLUMNS", include_tailing_empty=False, include_tailing_empty_rows=False, ) except (RefreshError, OAuthError, GoogleAuthError, UserAccessTokenError) as exc: raise AuthError(exc) except errors.HttpError as exc: if exc.resp.status == 403: raise PermissionError(exc) raise column_names, data = self.drop_extra_columns(data) data = self.fill_empty_cells(data) yield ExportContext( columns=column_names, data=data, data_orient=DataOrient.columns, request_kwargs={ "sheet_id": self.export.sheet_id, "sheet_name": sheet.title, "sheet_url": sheet.url, "page_id": self.export.page_id, "page_name": page.title, }, )
def __call__(self, start_period: dt.datetime, end_period: dt.datetime, **kwargs) -> Iterator[ExportContext]: try: params = self.collect_params(start_period, end_period) self.logger.info("Exportation data") report = self.client.stats().get(params=params) for result in report().pages(): self.logger.info("Iter export data") yield ExportContext( export_kwargs=result().request_kwargs, columns=report.columns, data=result().to_columns(), data_orient=DataOrient.columns, ) except YandexMetrikaTokenError as ex: raise AuthError(ex)
def __call__(self, start_period: dt.datetime, end_period: dt.datetime, **kwargs) -> Iterator[ExportContext]: for url_params, get_params in self.collect_params( start_period, end_period): method = getattr(self.client, self.resource) try: self.logger.info("Exportation data") result = method(**url_params).get(params=get_params) data = self.processing_response_data(self.resource, result().data, self.columns) yield ExportContext( export_kwargs=result().request_kwargs, columns=self.columns, data=data, data_orient=DataOrient.dict, ) except YandexMetrikaTokenError as ex: raise AuthError(ex)
def __call__(self, start_period: dt.datetime, end_period: dt.datetime, **kwargs) -> Iterator[Union[ExportContext, SleepIteration]]: from tapi_yandex_direct import exceptions self.logger.info("Exportation data") result = None page_iterator = None api_error_retries = 10 while True: try: if result is None: body = self.collect_params(start_period, end_period) method = getattr(self.client, self.export.resource) result = method().post(data=body) if self.export.resource != "reports": if page_iterator is None: page_iterator = result().pages( max_pages=kwargs.get("max_pages")) page = next(page_iterator) except exceptions.YandexDirectTokenError as exc: raise AuthError(exc) except exceptions.YandexDirectNotEnoughUnitsError: yield SleepIteration(sleep=60 * 5) continue except exceptions.YandexDirectRequestsLimitError: yield SleepIteration(sleep=10) continue except exceptions.YandexDirectClientError as exc: if api_error_retries and exc.error_code in (52, 1000, 1001, 1002): api_error_retries -= 1 yield SleepIteration(sleep=10) continue raise except ConnectionError: if api_error_retries: api_error_retries -= 1 yield SleepIteration(sleep=10) continue raise except StopIteration: break else: if self.export.resource == "reports": if result.status_code in (201, 202): result = None yield SleepIteration(sleep=10) continue data = result().to_values() yield ExportContext( request_kwargs=result.request_kwargs, columns=result.columns, data=data, data_orient=DataOrient.values, ) break else: self.logger.info("Iter export data") columns = [] if page.data: columns = sorted(page.data[0].keys()) yield ExportContext( request_kwargs=page.request_kwargs, columns=columns, data=page.data, data_orient=DataOrient.dict, )
def __call__( self, start_period: dt.datetime, end_period: dt.datetime, dry_run=False, **kwargs, ) -> Iterator[Union[ExportContext, SleepIteration]]: from tapi_yandex_metrika.exceptions import YandexMetrikaTokenError try: params = self.collect_params(start_period, end_period) self.logger.info("Exportation data") request_id = self.search_identic_report(start_period, end_period, params) if request_id is None: while True: # Evaluate report. result = self.client.evaluate().get(params=params) if result["log_request_evaluation"]["possible"] is False: sleeptime = 60 * 5 self.logger.info( f"The report store is full. " f"Waiting for when to be free {sleeptime} sec.") if dry_run is True: raise Exception("The report store is full") yield SleepIteration(sleep=sleeptime) else: # Create report. result = self.client.create().post(params=params) request_id = result["log_request"]["request_id"] break # Wait report. repeat_number = 1 while True: # Wait report. result = self.client.info(requestId=request_id).get() status = result["log_request"]["status"] if status == "processed": self.logger.info(f"Download report") report = self.client.download(requestId=request_id).get() break elif "cleaned" in status: raise Exception( "The report does not exist, it has been cleared. " "Curent report status is '{}'".format(status)) else: sleeptime = self.sleeptime(repeat_number) self.logger.info("Wait report {} sec.", sleeptime) if dry_run is True: time.sleep(10) yield SleepIteration(sleep=sleeptime) repeat_number += 1 # Download report. for part in report().parts(): self.logger.info("Iter export data") yield ExportContext( request_kwargs=part().request_kwargs, columns=report.columns, data=part().to_columns(), data_orient=DataOrient.columns, ) if dry_run is not True: self.client.clean(requestId=request_id).post() except YandexMetrikaTokenError as ex: raise AuthError(ex)
def __call__(self, start_period: dt.datetime, end_period: dt.datetime, **kwargs) -> Iterator[Union[ExportContext, SleepIteration]]: import criteo_marketing_transition as cm from criteo_marketing_transition.rest import ApiException from flowmaster.operators.etl import DataOrient self.logger.info("Exportation data") columns = self.export.params.dimensions + self.export.params.metrics stats_query_message = cm.StatisticsReportQueryMessage( dimensions=self.export.params.dimensions, metrics=self.export.params.metrics, start_date=start_period.date().isoformat(), end_date=end_period.date().isoformat(), currency=self.export.params.currency, format="json", ) iter_num = 0 api_error_retries = 0 while True: self.logger.info("Iter export data") iter_num += 1 try: ( response_content, http_code, response_headers, ) = self.analytics_api.get_adset_report_with_http_info( statistics_report_query_message=stats_query_message, async_req=True, _preload_content=False, ).get() except ApiException as exc: if (exc.status == 401 or exc.body.get("error") == "credentials_no_longer_supported"): raise AuthError(exc) if exc.status == 403: raise ForbiddenError(exc) if exc.status == 429: if api_error_retries: api_error_retries -= 1 # https://developers.criteo.com/marketing-solutions/docs/requesting-a-report yield SleepIteration(sleep=60) continue if exc.status in (500, 503): # https://developers.criteo.com/marketing-solutions/docs/how-to-handle-api-errors if api_error_retries: api_error_retries -= 1 yield SleepIteration(sleep=iter_num * 20) continue raise else: if http_code == 200: if response_content: response_content: HTTPResponse content: str = response_content.read().decode( "utf-8-sig") data = orjson.loads(content) if self.export.chunk_size: for chunk in chunker( data["Rows"], size=self.export.chunk_size, ): yield ExportContext( columns=columns, data=chunk, data_orient=DataOrient.dict, response_kwargs={ "headers": response_headers }, ) else: yield ExportContext( columns=columns, data=data["Rows"], data_orient=DataOrient.dict, response_kwargs={"headers": response_headers}, ) else: self.logger.warning("Didn't receive data") break else: if api_error_retries: api_error_retries -= 1 yield SleepIteration(sleep=10) raise Exception( f"CriteoError: code={http_code}, headers={response_headers}\n" f"response_content={response_content}")