def check_and_get_async_jobs(self, async_jobs): self.async_requests = [] for job in async_jobs: ar = AdReportRun(job['id']) report = self.get_report(ar) percent = report['async_percent_completion'] logging.info('FB async_job #{} percent done ' '{}%'.format(job['id'], percent)) if percent == 100 and (report['async_status'] == 'Job Completed'): try: complete_job = list(ar.get_result()) except FacebookRequestError as e: self.request_error(e) self.async_requests.append(job) complete_job = None except FacebookBadObjectError as e: logging.warning('Facebook Bad Object Error: {}'.format(e)) self.async_requests.append(job) complete_job = None if complete_job: self.df = self.df.append(complete_job, ignore_index=True) else: self.async_requests.append(job) if self.async_requests: time.sleep(30) self.check_and_get_async_jobs(self.async_requests)
def iterate_edge_async(self, target_objects_class, fields=None, params=None, is_async=False, include_summary=True, endpoint=None): from facebook_business.adobjects.adreportrun import AdReportRun """ Behaves as iterate_edge(...) if parameter is_async if False (Default value) If is_async is True: Returns an AsyncJob which can be checked using remote_read() to verify when the job is completed and the result ready to query or download using get_result() Example: >>> job = object.iterate_edge_async( TargetClass, fields, params, is_async=True) >>> time.sleep(10) >>> job.remote_read() >>> if job: result = job.read_result() print result """ synchronous = not is_async synchronous_iterator = self.iterate_edge( target_objects_class, fields, params, fetch_first_page=synchronous, include_summary=include_summary, ) if synchronous: return synchronous_iterator if not params: params = {} else: params = dict(params) self.__class__._assign_fields_to_params(fields, params) # To force an async response from an edge, do a POST instead of GET. # The response comes in the format of an AsyncJob which # indicates the progress of the async request. if endpoint is None: endpoint = target_objects_class.get_endpoint() response = self.get_api_assured().call( 'POST', (self.get_id_assured(), endpoint), params=params, ).json() # AsyncJob stores the real iterator # for when the result is ready to be queried result = AdReportRun() if 'report_run_id' in response: response['id'] = response['report_run_id'] result._set_data(response) return result
def _facebook_report( self, account_id: str, api: FacebookAdsApi, params: Dict[str, Any], fields: List[str], sleep_time: int = 5, ) -> List[AdsInsights]: """ Pulls data from the Facebook Ads API with given account_id :param account_id: Facebook Account ID that holds ads information https://developers.facebook.com/docs/marketing-api/reference/ads-insights/ :type account_id: str :param api: FacebookAdsApi created in the hook :type api: FacebookAdsApi :param fields: List of fields that is obtained from Facebook. Found in AdsInsights.Field class. https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0 :type fields: List[str] :param params: Parameters that determine the query for Facebook https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0 :type params: Dict[str, Any] :param sleep_time: Time to sleep when async call is happening :type sleep_time: int """ ad_account = AdAccount(account_id, api=api) _async = ad_account.get_insights(params=params, fields=fields, is_async=True) while True: request = _async.api_get() async_status = request[AdReportRun.Field.async_status] percent = request[AdReportRun.Field.async_percent_completion] self.log.info("%s %s completed, async_status: %s", percent, "%", async_status) if async_status == JobStatus.COMPLETED.value: self.log.info("Job run completed") break if async_status in [ JobStatus.SKIPPED.value, JobStatus.FAILED.value ]: message = f"{async_status}. Please retry." raise AirflowException(message) time.sleep(sleep_time) report_run_id = _async.api_get()["report_run_id"] report_object = AdReportRun(report_run_id, api=api) self.log.info("Extracting data from returned Facebook Ads Iterators") insights = report_object.get_insights() return list(insights)
def bulk_facebook_report( self, params: Dict[str, Any], fields: List[str], sleep_time: int = 5, ) -> List[AdsInsights]: """ Pulls data from the Facebook Ads API :param fields: List of fields that is obtained from Facebook. Found in AdsInsights.Field class. https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0 :type fields: List[str] :param params: Parameters that determine the query for Facebook https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0 :type fields: Dict[str, Any] :param sleep_time: Time to sleep when async call is happening :type sleep_time: int :return: Facebook Ads API response, converted to Facebook Ads Row objects :rtype: List[AdsInsights] """ api = self._get_service() ad_account = AdAccount(api.get_default_account_id(), api=api) _async = ad_account.get_insights(params=params, fields=fields, is_async=True) while True: request = _async.api_get() async_status = request[AdReportRun.Field.async_status] percent = request[AdReportRun.Field.async_percent_completion] self.log.info("%s %s completed, async_status: %s", percent, "%", async_status) if async_status == JobStatus.COMPLETED.value: self.log.info("Job run completed") break if async_status in [ JobStatus.SKIPPED.value, JobStatus.FAILED.value ]: message = "{async_status}. Please retry.".format( async_status=async_status) raise AirflowException(message) time.sleep(sleep_time) report_run_id = _async.api_get()["report_run_id"] report_object = AdReportRun(report_run_id, api=api) insights = report_object.get_insights() self.log.info("Extracting data from returned Facebook Ads Iterators") return list(insights)
def check_and_get_async_jobs(self, async_jobs): self.async_requests = [] for fb_request in async_jobs: try: job = fb_request.insights except AttributeError as e: logging.warning( 'A FB async_job does not contain insights and will ' 'be requested again. This is request #{} Error: {}'. format(fb_request.times_requested, e)) self.reset_report_request(fb_request) continue ar = AdReportRun(job['id']) report = self.get_report(ar) percent = report['async_percent_completion'] need_reset = fb_request.check_last_percent(percent) if need_reset: logging.warning( 'FB async_job #{} has been stuck for {} attempts and will ' 'be requested again. This is request #{}'.format( job['id'], fb_request.times_requested * 10, fb_request.times_requested)) self.reset_report_request(fb_request) continue logging.info('FB async_job #{} percent done ' '{}%'.format(job['id'], percent)) if percent == 100 and (report['async_status'] == 'Job Completed'): try: complete_job = list(ar.get_result()) except FacebookRequestError as e: self.request_error(e) self.async_requests.append(job) complete_job = None except FacebookBadObjectError as e: logging.warning('Facebook Bad Object Error: {}'.format(e)) self.async_requests.append(job) complete_job = None if complete_job: self.df = self.df.append(complete_job, ignore_index=True) fb_request.complete = True else: self.async_requests.append(fb_request) if self.async_requests: time.sleep(30) self.check_and_get_async_jobs(self.async_requests)
def adreport_fixture(mocker, api): ao = AdReportRun(fbid="123", api=api) ao["report_run_id"] = "123" mocker.patch.object(ao, "api_get", side_effect=ao.api_get) mocker.patch.object(ao, "get_result", side_effect=ao.get_result) return ao
self.__class__._assign_fields_to_params(fields, params) # To force an async response from an edge, do a POST instead of GET. # The response comes in the format of an AsyncJob which # indicates the progress of the async request. if endpoint is None: endpoint = target_objects_class.get_endpoint() response = self.get_api_assured().call( 'POST', (self.get_id_assured(), endpoint), params=params, ).json() # AsyncJob stores the real iterator # for when the result is ready to be queried result = AdReportRun() if 'report_run_id' in response: response['id'] = response['report_run_id'] result._set_data(response) return result def edge_object(self, target_objects_class, fields=None, params=None, endpoint=None): """ Returns first object when iterating over Cursor with argument self as source_object and the rest as given __init__ arguments. """ params = {} if not params else params.copy() params['limit'] = '1' for obj in self.iterate_edge( target_objects_class,