def __init__(self, request: FacebookRequest,
request_error: FacebookRequestError):
super().__init__(
description=request_error.api_error_message(),
# Generally 500 responses have no is_transient field in payload, even though they are transient
# in nature.
is_transient=request_error.api_transient_error()
or request_error.http_status() == 500,
data=request_error.body(),
)
self.request = request
self.request_error = request_error
def handle_call_rate_response(exc: FacebookRequestError) -> bool:
pause_time = DEFAULT_SLEEP_INTERVAL
platform_header = exc.http_headers().get("x-app-usage") or exc.http_headers().get("x-ad-account-usage")
if platform_header:
platform_header = json.loads(platform_header)
call_count = platform_header.get("call_count") or platform_header.get("acc_id_util_pct")
if call_count and call_count > 99:
logger.info(f"Reached platform call limit: {exc}")
buc_header = exc.http_headers().get("x-business-use-case-usage")
buc_header = json.loads(buc_header) if buc_header else {}
for business_object_id, stats in buc_header.items():
if stats.get("call_count", 0) > 99:
logger.info(f"Reached call limit on {stats['type']}: {exc}")
pause_time = max(pause_time, stats["estimated_time_to_regain_access"])
logger.info(f"Sleeping for {pause_time.total_seconds()} seconds")
sleep(pause_time.total_seconds())
return True
def error(self):
"""
Returns a FacebookRequestError (located in the exceptions module) with
an appropriate debug message.
"""
if self.is_failure():
return FacebookRequestError(
"Call was not successful",
self._call,
self.status(),
self.headers(),
self.body(),
)
else:
return None
def execute(self):
params = copy.deepcopy(self._params)
if self._api_type == "EDGE" and self._method == "GET":
cursor = Cursor(
target_objects_class=self._target_class,
params=params,
fields=self._fields,
include_summary=self._include_summary,
api=self._api,
node_id=self._node_id,
endpoint=self._endpoint,
)
cursor.load_next_page()
return cursor
if self._fields:
params['fields'] = ','.join(self._fields)
with open_files(self._file_params) as files:
response = self._api.call(
method=self._method,
path=(self._path),
params=params,
files=files,
api_version=self._api_version,
)
if response.error():
raise response.error()
if self._response_parser:
try:
return self._response_parser.parse_single(response.json())
except TypeError as te:
if "string indices must be integers" in str(te):
raise FacebookRequestError(
"Erroneous response from Facebook API",
response._call,
response.status(),
response.headers(),
response.body(),
)
else:
raise
else:
return response
def test_reported_task_on_failure_facebook_error(mock_get_status_and_bucket,
mock_notify, mock_report,
mock_from_job_scope):
exc = FacebookRequestError('test', {}, 404, [], '')
mock_job_scope = Mock(token='token')
mock_get_status_and_bucket.return_value = (
ExternalPlatformJobStatus.UserThrottlingError,
FailureBucket.UserThrottling,
)
@reported_task
def test_task(*_, **__):
raise exc
test_task(mock_job_scope)
assert mock_job_scope.running_time is not None
assert mock_report.delay.call_args_list == [
call(ExternalPlatformJobStatus.UserThrottlingError, mock_job_scope)
]
assert not mock_notify.called
mock_from_job_scope.return_value.report_usage_per_failure_bucket.assert_called_once_with(
'token', FailureBucket.UserThrottling)
class FacebookExecutorTest(unittest.TestCase):
""" Facebook Executor Test """
def test_transform_campaign_budget(self):
"""
Testing transform_campaign_budget() using get_campaigns_mock().
"""
mock_data = get_campaigns_mock()
result = transform_campaign_budget(mock_data)
assert_frame_equal(result,
transform_campaign_budget_expected_outcome())
def test_build_predicted_revenue_events(self):
"""
Testing build_predicted_revenue_events() using get_predicted_revenue_mock().
"""
predicted_revenue_events = get_predicted_revenue_mock()
result = build_predicted_revenue_events(predicted_revenue_events,
'Predicted Revenue')
df_result = result[1]
assert_series_equal(predicted_revenue_events["date"],
df_result["date_source"],
check_names=False)
assert_series_equal(predicted_revenue_events["predicted_revenue"],
df_result["predicted_revenue"])
assert_series_equal(predicted_revenue_events["currency"],
df_result["currency"])
assert_series_equal(predicted_revenue_events["facebook_browser_id"],
df_result["facebook_browser_id"])
assert_series_equal(predicted_revenue_events["shop"],
df_result["shop"])
def test_calculate_batches(self):
"""
Testing build_predicted_revenue_events() using get_predicted_revenue_mock().
"""
result = calculate_batches(10, 3)
res_even = calculate_batches(10, 5)
res_single_event = calculate_batches(1, 5)
res_no_event = calculate_batches(0, 5)
self.assertEqual(result, 4)
self.assertEqual(res_even, 2)
self.assertEqual(res_single_event, 1)
self.assertEqual(res_no_event, 0)
def test_split_events_to_batches(self):
"""
Testing build_predicted_revenue_events() using get_predicted_revenue_mock().
"""
predicted_revenue_events = get_predicted_revenue_mock()
total_events = len(predicted_revenue_events)
batch_size = 2
batches = calculate_batches(total_events, batch_size)
result = split_events_to_batches(predicted_revenue_events, batch_size)
self.assertEqual(len(result), batches)
@mock.patch('pygyver.etl.facebook.FacebookExecutor.set_api_config',
side_effect=fb_login_mock)
@mock.patch(
'facebook_business.adobjects.serverside.event_request.EventRequest.execute',
side_effect=fb_api_mock)
def test_push_conversions_api_events(self, fb_api_mock, fb_login_mock):
"""
Testing push_conversions_api_events() using get_predicted_revenue_mock().
"""
fbe = FacebookExecutor()
fbe.set_pixel_id('1530331220624093')
predicted_revenue_events = get_predicted_revenue_mock()
events, log = build_predicted_revenue_events(predicted_revenue_events,
'Predicted Revenue')
result = fbe.push_conversions_api_events(events, 'TEST24777')
self.assertEqual(result['status'], 'API Success')
self.assertEqual(result['total_events'], len(predicted_revenue_events))
@mock.patch('pygyver.etl.facebook.FacebookExecutor.set_api_config',
side_effect=fb_login_mock)
@mock.patch(
'facebook_business.adobjects.serverside.event_request.EventRequest.execute',
side_effect=FacebookRequestError(message="test exception",
request_context=context,
http_status="404",
http_headers="some/headers",
body=error_json))
def test_push_conversions_api_events_error(self, fb_api_mock,
fb_login_mock):
"""
Testing push_conversions_api_events() using get_predicted_revenue_mock().
"""
fbe = FacebookExecutor()
fbe.set_pixel_id('1530331220624093')
predicted_revenue_events = get_predicted_revenue_mock()
events, log = build_predicted_revenue_events(predicted_revenue_events,
'Predicted Revenue')
result = fbe.push_conversions_api_events(events, 'TEST24777')
self.assertEqual(result['status'], 'API Error')
def should_retry_api_error(exc: FacebookRequestError):
# Retryable OAuth Error Codes
if exc.api_error_type() == "OAuthException" and exc.api_error_code(
) in (1, 2, 4, 17, 341, 368):
return True
# Rate Limiting Error Codes
if exc.api_error_code() in (4, 17, 32, 613):
return True
if exc.http_status() == status_codes.TOO_MANY_REQUESTS:
return True
# FIXME: add type and http_status
if exc.api_error_code() == 10 and exc.api_error_message(
) == "(#10) Not enough viewers for the media to show insights":
return False # expected error
# Issue 4028, Sometimes an error about the Rate Limit is returned with a 400 HTTP code
if exc.http_status(
) == status_codes.BAD_REQUEST and exc.api_error_code(
) == 100 and exc.api_error_subcode() == 33:
return True
if exc.api_transient_error():
return True
# FIXME: add type, code and http_status
if exc.api_error_subcode() == 2108006:
return False
return False
def get_data_from_api(start_date,end_date):
Campaign_ID = []
Page_ID = []
Amount_Spent = []
# Page_Name = []
campaigns_all = {"Campaign_id":[],
"Page_id":[],
"Amount_spent":[],
}
page_id = None
pixel_id = None
access_token = 'EAAUbBhxccoEBALbQCDsVMLzwdZBZAZBXApZA0t1Qy3GtjZALfs89EMFhH62f5Kp7FWvshYRTrur41B14vICAgTf1TOba8qx7SBPejdqR4gZBqZCGDo0l0WvsmzubUKKqHncpyqhSpUqcO7O0WJsB1PnSZAMY7t7awukDuIYwrisTYwZDZD'
bussiness_account_id = '1517651558352959'
app_secret = '7a3ad485c97dbf45ee83562bc0dcb570'
app_id = '1437087943127681'
start_time = datetime.datetime.now()
FacebookAdsApi.init(app_id, app_secret, access_token)
business = Business(bussiness_account_id)
# Get all ad accounts on the business account
my_ad_account =business.get_owned_ad_accounts(fields=[AdAccount.Field.id])
# fields = [
# 'name',
# 'objective',
# # 'spend',
# ]
# params = {
# 'time_range':{'since':start_date,'until':end_date},
# # 'date_preset':'yesterday',
# 'effective_status': ['ACTIVE','PAUSED'],
# }
# Iterate through the adaccounts
for account in my_ad_account:
print(len(my_ad_account))
# i = 0
# Create an addaccount object from the adaccount id to make it possible to get insights
tempaccount = AdAccount(account[AdAccount.Field.id])
# campaigns_iter = tempaccount.get_campaigns(fields = fields, params = params)
# CAMPAIGN_UPDATE_BATCH_LIMIT = 5
# for campaigns in generate_batches(campaigns_iter,CAMPAIGN_UPDATE_BATCH_LIMIT):
# api_batch = api.new_batch()
# for i, campaign in enumerate(campaigns_iter):
# adset = AdAccount(campaign[Campaign.Field.id]).get_ad_sets(
# fields=['id', 'name', 'promoted_object'],
# params = {})
# print(adset)
# api_batch.execute()
# spend_val.append(campaign[Campaign.Field.id])
# print(campaign, i)
# print(spend_val)
# print(set(spend_val))
# Grab insight info for all ads in the adaccount
account_data = tempaccount.get_insights(params={
# 'time_increment':'1',
'time_range':{'since':start_date, 'until':end_date},
'level':'campaign',
},
fields=[
AdsInsights.Field.campaign_id,
AdsInsights.Field.campaign_name,
AdsInsights.Field.spend,
]
)
for campaign in account_data:
try:
#Check if you reached 75% of the limit, if yes then back-off for 5 minutes (put this chunk in your 'for ad is ads' loop, every 100-200 iterations)
# if (check_limit(bussiness_account_id,access_token)>75):
# print('75% Rate Limit Reached. Cooling Time 5 Minutes.')
# logging.debug('75% Rate Limit Reached. Cooling Time 5 Minutes.')
# time.sleep(300)
#ID OF Campaign
# if campaign!=[]:
# print(campaign)
# print(len(account_data))
campaign_id = campaign[AdsInsights.Field.campaign_id]
campaign_spent_val = campaign[AdsInsights.Field.spend]
# print(campaign_spent_val)
my_camp = Campaign(campaign_id)
print(my_camp)
#Campaign Insights Object
# campaign_spent_obj = my_camp.get_insights(params={}, fields=[AdsInsights.Field.spend])
# campaign_spent = campaign_spent_obj[Campaign.Field.spend]
# print(campaign_spent_obj)
#Campaign Spend value
# campaigns_all["Amount_spent"].append(campaign_spent_val)
#AdSet Object
adset = AdAccount(my_camp[Campaign.Field.id]).get_ad_sets(
fields=['id', 'name', 'promoted_object'],
params = {})
#page and Pixel ID from Adset
if 'page_id' in adset[0][AdSet.Field.promoted_object]:
page_id = adset[0][AdSet.Field.promoted_object][AdPromotedObject.Field.page_id]
campaigns_all["Page_id"].append(page_id)
Page_ID.append(page_id)
# page_req = rq.head('https://facebook.com/'+page_id)
# print(page_req.headers)
# # for page in Page_ID:
# print(Page(page_id).api_get(fields=['name'],params={}))
elif 'pixel_id' in adset[0][AdSet.Field.promoted_object]:
pixel_id = adset[0][AdSet.Field.promoted_object][AdPromotedObject.Field.pixel_id]
campaigns_all["Page_id"].append(pixel_id)
Page_ID.append(pixel_id)
else:
continue
# Add Values to Dictionary
campaigns_all["Campaign_id"].append(campaign_id)
campaigns_all["Amount_spent"].append(campaign_spent_val)
Campaign_ID.append(campaign_id)
Amount_Spent.append(campaign_spent_val)
# print(campaigns_all)
time.sleep(2)
except KeyError as e:
print(e)
continue
except Exception as e:
print(e)
if FacebookRequestError.api_error_code(e) == 17:
print(campaigns_all)
print("Limit Reached")
print("Wait 5 minutes for cooldown")
time.sleep(300)
continue
finally:
end_time = datetime.datetime.now()
diff = end_time - start_time
print(diff)
tuples_of_data = list(zip(Campaign_ID,Page_ID,Amount_Spent))
sum_amount = compare_values(tuples_of_data)
print(sum_amount)
# print(diff.total_seconds())
return campaigns_all,sum_amount
def _get_exception_description(self, e: FacebookRequestError):
if e.body() is not None and 'error' in e.body(
) and 'message' in e.body()['error']:
return e.body()['error']['message']
return "no descriptions"
def launch_job(self):
"""
1. Calls POST to create job
2. Creates and store in attributes AsyncAioJob
3. Puts self in futures
:return: None
"""
# To force an async response from an edge, do a POST instead of GET.
# The response comes in the format of an AsyncAioJob which
# indicates the progress of the async request.
response = {}
for i in range(5):
# TODO: refactor this into async schema like in regular AioEdgeIterator
try:
response = self._source_object.get_api_assured().call(
'POST',
(self._source_object.get_id_assured(),
self._target_objects_class.get_endpoint()),
params=self.params,
).json()
except FacebookRequestError as exc:
if i < 4 and (exc.api_error_code() in [
FacebookErrorCodes.unknown,
FacebookErrorCodes.temporary
] or not exc.is_body_json()):
time.sleep(15 + i * 15)
elif i < 4 and exc.api_error_code() in (
FacebookErrorCodes.rate_limit,
FacebookErrorCodes.too_many_requests):
time.sleep(60 + i * 60)
else:
raise exc
except ConnectionError as exc:
if i < 4:
time.sleep(10 + i * 10)
else:
raise exc
else:
if isinstance(response, string_types) and i < 4:
time.sleep(15 + i * 15)
else:
if isinstance(response, string_types):
raise FacebookRequestError(
"Facebook response is a string", {
"method":
"POST",
"path":
"/{}/{}".format(
self._source_object.get_id_assured(),
self._target_objects_class.get_endpoint()),
"params":
self.params
}, 500, {}, response)
break
self.job_started_at = time.time()
self.attempt += 1
self.failed_attempt = 0
if 'report_run_id' in response:
response['id'] = response['report_run_id']
# AsyncAioJob stores the real iterator
# for when the result is ready to be queried
self.job = AsyncAioJob(self._target_objects_class,
edge_params=self.params,
has_action=self.has_action,
needs_action_device=self.needs_action_device,
has_filters=self.has_filters,
needs_carousel_name=self.needs_carousel_name)
self.job._set_data(response)
self.job_id = response['id'] if 'id' in response else 'no id'
self._source_object.get_api_assured().put_in_futures(self)
logger.debug('started a job, job_id: {}'.format(
response['id'] if 'id' in response else 'no id'))
self.job_last_completion_change_time = time.time()
self.job_previous_completion_value = 0
def should_retry_api_error(exc: FacebookRequestError):
# Retryable OAuth Error Codes
if exc.api_error_type() == "OAuthException" and exc.api_error_code(
) in (1, 2, 4, 17, 341, 368):
return True
# Rate Limiting Error Codes
if exc.api_error_code() in (4, 17, 32, 613):
return True
if exc.http_status() == status_codes.TOO_MANY_REQUESTS:
return True
if (exc.api_error_type() == "OAuthException"
and exc.api_error_code() == 10 and exc.api_error_message()
== "(#10) Not enough viewers for the media to show insights"):
return True
# Issue 4028, Sometimes an error about the Rate Limit is returned with a 400 HTTP code
if exc.http_status(
) == status_codes.BAD_REQUEST and exc.api_error_code(
) == 100 and exc.api_error_subcode() == 33:
return True
if exc.api_transient_error():
return True
# The media was posted before the most recent time that the user's account
# was converted to a business account from a personal account.
if exc.api_error_type() == "OAuthException" and exc.api_error_code(
) == 100 and exc.api_error_subcode() == 2108006:
return True
return False
