def check_and_get_async_jobs(self, async_jobs):
self.async_requests = []
for job in async_jobs:
ar = AdReportRun(job['id'])
report = self.get_report(ar)
percent = report['async_percent_completion']
logging.info('FB async_job #{} percent done '
'{}%'.format(job['id'], percent))
if percent == 100 and (report['async_status'] == 'Job Completed'):
try:
complete_job = list(ar.get_result())
except FacebookRequestError as e:
self.request_error(e)
self.async_requests.append(job)
complete_job = None
except FacebookBadObjectError as e:
logging.warning('Facebook Bad Object Error: {}'.format(e))
self.async_requests.append(job)
complete_job = None
if complete_job:
self.df = self.df.append(complete_job, ignore_index=True)
else:
self.async_requests.append(job)
if self.async_requests:
time.sleep(30)
self.check_and_get_async_jobs(self.async_requests)
def iterate_edge_async(self,
target_objects_class,
fields=None,
params=None,
is_async=False,
include_summary=True,
endpoint=None):
from facebook_business.adobjects.adreportrun import AdReportRun
"""
Behaves as iterate_edge(...) if parameter is_async if False
(Default value)
If is_async is True:
Returns an AsyncJob which can be checked using remote_read()
to verify when the job is completed and the result ready to query
or download using get_result()
Example:
>>> job = object.iterate_edge_async(
TargetClass, fields, params, is_async=True)
>>> time.sleep(10)
>>> job.remote_read()
>>> if job:
result = job.read_result()
print result
"""
synchronous = not is_async
synchronous_iterator = self.iterate_edge(
target_objects_class,
fields,
params,
fetch_first_page=synchronous,
include_summary=include_summary,
)
if synchronous:
return synchronous_iterator
if not params:
params = {}
else:
params = dict(params)
self.__class__._assign_fields_to_params(fields, params)
# To force an async response from an edge, do a POST instead of GET.
# The response comes in the format of an AsyncJob which
# indicates the progress of the async request.
if endpoint is None:
endpoint = target_objects_class.get_endpoint()
response = self.get_api_assured().call(
'POST',
(self.get_id_assured(), endpoint),
params=params,
).json()
# AsyncJob stores the real iterator
# for when the result is ready to be queried
result = AdReportRun()
if 'report_run_id' in response:
response['id'] = response['report_run_id']
result._set_data(response)
return result
def _facebook_report(
self,
account_id: str,
api: FacebookAdsApi,
params: Dict[str, Any],
fields: List[str],
sleep_time: int = 5,
) -> List[AdsInsights]:
"""
Pulls data from the Facebook Ads API with given account_id
:param account_id: Facebook Account ID that holds ads information
https://developers.facebook.com/docs/marketing-api/reference/ads-insights/
:type account_id: str
:param api: FacebookAdsApi created in the hook
:type api: FacebookAdsApi
:param fields: List of fields that is obtained from Facebook. Found in AdsInsights.Field class.
https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0
:type fields: List[str]
:param params: Parameters that determine the query for Facebook
https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0
:type params: Dict[str, Any]
:param sleep_time: Time to sleep when async call is happening
:type sleep_time: int
"""
ad_account = AdAccount(account_id, api=api)
_async = ad_account.get_insights(params=params,
fields=fields,
is_async=True)
while True:
request = _async.api_get()
async_status = request[AdReportRun.Field.async_status]
percent = request[AdReportRun.Field.async_percent_completion]
self.log.info("%s %s completed, async_status: %s", percent, "%",
async_status)
if async_status == JobStatus.COMPLETED.value:
self.log.info("Job run completed")
break
if async_status in [
JobStatus.SKIPPED.value, JobStatus.FAILED.value
]:
message = f"{async_status}. Please retry."
raise AirflowException(message)
time.sleep(sleep_time)
report_run_id = _async.api_get()["report_run_id"]
report_object = AdReportRun(report_run_id, api=api)
self.log.info("Extracting data from returned Facebook Ads Iterators")
insights = report_object.get_insights()
return list(insights)
def bulk_facebook_report(
self,
params: Dict[str, Any],
fields: List[str],
sleep_time: int = 5,
) -> List[AdsInsights]:
"""
Pulls data from the Facebook Ads API
:param fields: List of fields that is obtained from Facebook. Found in AdsInsights.Field class.
https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0
:type fields: List[str]
:param params: Parameters that determine the query for Facebook
https://developers.facebook.com/docs/marketing-api/insights/parameters/v6.0
:type fields: Dict[str, Any]
:param sleep_time: Time to sleep when async call is happening
:type sleep_time: int
:return: Facebook Ads API response, converted to Facebook Ads Row objects
:rtype: List[AdsInsights]
"""
api = self._get_service()
ad_account = AdAccount(api.get_default_account_id(), api=api)
_async = ad_account.get_insights(params=params,
fields=fields,
is_async=True)
while True:
request = _async.api_get()
async_status = request[AdReportRun.Field.async_status]
percent = request[AdReportRun.Field.async_percent_completion]
self.log.info("%s %s completed, async_status: %s", percent, "%",
async_status)
if async_status == JobStatus.COMPLETED.value:
self.log.info("Job run completed")
break
if async_status in [
JobStatus.SKIPPED.value, JobStatus.FAILED.value
]:
message = "{async_status}. Please retry.".format(
async_status=async_status)
raise AirflowException(message)
time.sleep(sleep_time)
report_run_id = _async.api_get()["report_run_id"]
report_object = AdReportRun(report_run_id, api=api)
insights = report_object.get_insights()
self.log.info("Extracting data from returned Facebook Ads Iterators")
return list(insights)
def check_and_get_async_jobs(self, async_jobs):
self.async_requests = []
for fb_request in async_jobs:
try:
job = fb_request.insights
except AttributeError as e:
logging.warning(
'A FB async_job does not contain insights and will '
'be requested again. This is request #{} Error: {}'.
format(fb_request.times_requested, e))
self.reset_report_request(fb_request)
continue
ar = AdReportRun(job['id'])
report = self.get_report(ar)
percent = report['async_percent_completion']
need_reset = fb_request.check_last_percent(percent)
if need_reset:
logging.warning(
'FB async_job #{} has been stuck for {} attempts and will '
'be requested again. This is request #{}'.format(
job['id'], fb_request.times_requested * 10,
fb_request.times_requested))
self.reset_report_request(fb_request)
continue
logging.info('FB async_job #{} percent done '
'{}%'.format(job['id'], percent))
if percent == 100 and (report['async_status'] == 'Job Completed'):
try:
complete_job = list(ar.get_result())
except FacebookRequestError as e:
self.request_error(e)
self.async_requests.append(job)
complete_job = None
except FacebookBadObjectError as e:
logging.warning('Facebook Bad Object Error: {}'.format(e))
self.async_requests.append(job)
complete_job = None
if complete_job:
self.df = self.df.append(complete_job, ignore_index=True)
fb_request.complete = True
else:
self.async_requests.append(fb_request)
if self.async_requests:
time.sleep(30)
self.check_and_get_async_jobs(self.async_requests)
def adreport_fixture(mocker, api):
ao = AdReportRun(fbid="123", api=api)
ao["report_run_id"] = "123"
mocker.patch.object(ao, "api_get", side_effect=ao.api_get)
mocker.patch.object(ao, "get_result", side_effect=ao.get_result)
return ao
self.__class__._assign_fields_to_params(fields, params)
# To force an async response from an edge, do a POST instead of GET.
# The response comes in the format of an AsyncJob which
# indicates the progress of the async request.
if endpoint is None:
endpoint = target_objects_class.get_endpoint()
response = self.get_api_assured().call(
'POST',
(self.get_id_assured(), endpoint),
params=params,
).json()
# AsyncJob stores the real iterator
# for when the result is ready to be queried
result = AdReportRun()
if 'report_run_id' in response:
response['id'] = response['report_run_id']
result._set_data(response)
return result
def edge_object(self, target_objects_class, fields=None, params=None, endpoint=None):
"""
Returns first object when iterating over Cursor with argument
self as source_object and the rest as given __init__ arguments.
"""
params = {} if not params else params.copy()
params['limit'] = '1'
for obj in self.iterate_edge(
target_objects_class,