def test_resource_removed_fail(self):
openstack_utils.resource_reaches_status.retry.wait = \
tenacity.wait_none()
resource_mock = mock.MagicMock()
resource_mock.list.return_value = [mock.MagicMock(id='e01df65a')]
with self.assertRaises(AssertionError):
openstack_utils.resource_removed(resource_mock, 'e01df65a')
def test_ping_response_fail(self):
openstack_utils.ping_response.retry.wait = \
tenacity.wait_none()
self.patch_object(openstack_utils.subprocess, 'check_call')
self.check_call.side_effect = Exception()
with self.assertRaises(Exception):
openstack_utils.ping_response('10.0.0.10')
def test_resource_reaches_status_fail(self):
openstack_utils.resource_reaches_status.retry.wait = \
tenacity.wait_none()
resource_mock = mock.MagicMock()
resource_mock.get.return_value = mock.MagicMock(status='unavailable')
with self.assertRaises(AssertionError):
openstack_utils.resource_reaches_status(resource_mock, 'e01df65a')
def test_wait_arbitrary_sum(self):
r = Retrying(wait=sum([tenacity.wait_fixed(1),
tenacity.wait_random(0, 3),
tenacity.wait_fixed(5),
tenacity.wait_none()]))
# Test it a few time since it's random
for i in six.moves.range(1000):
w = r.wait(1, 5)
self.assertLess(w, 9)
self.assertGreaterEqual(w, 6)
def test_request_fail(self, mock_req):
mock_req.side_effect = requests.ConnectTimeout(*["Exception message"])
# disable waiting for test
self.bitfinex_resp.make_request.retry.wait = wait_none()
with self.assertRaises(Exception) as e:
req = self.bitfinex_resp.make_request(url='url')
self.assertEqual(mock_req.call_count, 5)
def test_run_with_advanced_retry(self, m):
m.get(u'http://test:8080/v1/test', status_code=200, reason=u'OK')
retry_args = dict(wait=tenacity.wait_none(),
stop=tenacity.stop_after_attempt(3),
retry=tenacity.retry_if_exception_type(Exception),
reraise=True)
with mock.patch('airflow.hooks.base_hook.BaseHook.get_connection',
side_effect=get_airflow_connection):
response = self.get_hook.run_with_advanced_retry(
endpoint='v1/test', _retry_args=retry_args)
self.assertIsInstance(response, requests.Response)
def test_retry_on_conn_error(self, mocked_session):
retry_args = dict(
wait=tenacity.wait_none(),
stop=tenacity.stop_after_attempt(7),
retry=tenacity.retry_if_exception_type(requests.exceptions.ConnectionError),
)
def send_and_raise(unused_request, **kwargs):
raise requests.exceptions.ConnectionError
mocked_session().send.side_effect = send_and_raise
# The job failed for some reason
with pytest.raises(tenacity.RetryError):
self.get_hook.run_with_advanced_retry(endpoint='v1/test', _retry_args=retry_args)
assert self.get_hook._retry_obj.stop.max_attempt_number + 1 == mocked_session.call_count
def test_connection_retry(self, traptor):
"""Ensure we can correctly retry connection failures, throttling, and auth failures."""
traptor.logger = MagicMock()
# Trade out the wait policy for the unit test
traptor._create_birdy_stream.retry.wait = wait_none()
if traptor.traptor_type == 'follow':
traptor._create_twitter_follow_stream = MagicMock(side_effect=None)
try:
traptor._create_birdy_stream()
except Exception as e:
# Follow should not throw exception
raise e
assert traptor._create_twitter_follow_stream.call_count == 1
elif traptor.traptor_type == 'track':
traptor._create_twitter_track_stream = MagicMock(side_effect=[
TwitterApiError("api error 1"),
TwitterApiError("api error 2"),
TwitterApiError("api error 3"),
TwitterApiError("api error 4"),
None # Finally succeed
])
try:
traptor._create_birdy_stream()
except Exception as e:
# Track should not throw exception
raise e
assert traptor._create_twitter_track_stream.call_count == 5
elif traptor.traptor_type == 'locations':
traptor._create_twitter_locations_stream = MagicMock(
side_effect=TwitterAuthError("auth error"))
try:
traptor._create_birdy_stream()
except TwitterAuthError as e:
# Locations should throw TwitterAuthError
pass
assert traptor._create_twitter_locations_stream.call_count == 1
def test__stage_files_to_be_validated__gives_up_after_5_tries(
self, mock_download_file):
mock_download_file.side_effect = self._mock_stage_file_never_succeeding
with TemporaryDirectory() as staging_dir:
harness = ValidatorHarness(
path_to_validator=None,
s3_urls_of_files_to_be_validated=[self.s3_url],
staging_folder=staging_dir)
harness._stage_files_to_be_validated.retry.wait = tenacity.wait_none(
) # Speed things up
self.expected_file_path = f"{staging_dir}/{self.upload_area_id}/{self.filename}"
self.mock_download_file_call_count = 0
with self.assertRaises(UploadException):
harness._stage_files_to_be_validated()
self.assertEqual(5, self.mock_download_file_call_count)
def test__stage_files_to_be_validated__retries_5_times(
self, mock_download_file):
mock_download_file.side_effect = self._mock_stage_file_succeeding_on_the_5th_try
with TemporaryDirectory() as staging_dir:
harness = ValidatorHarness(
path_to_validator=None,
s3_urls_of_files_to_be_validated=[self.s3_url],
staging_folder=staging_dir)
harness._stage_files_to_be_validated.retry.wait = tenacity.wait_none(
) # Speed things up
self.expected_file_path = f"{staging_dir}/{self.upload_area_id}/{self.filename}"
self.mock_download_file_call_count = 0
harness._stage_files_to_be_validated()
self.assertEqual(5, self.mock_download_file_call_count)
self.assertTrue(os.path.isfile(self.expected_file_path))
def test_retry_on_conn_error(self, mocked_session):
retry_args = dict(
wait=tenacity.wait_none(),
stop=tenacity.stop_after_attempt(7),
retry=requests.exceptions.ConnectionError
)
def send_and_raise(request, **kwargs):
raise requests.exceptions.ConnectionError
mocked_session().send.side_effect = send_and_raise
# The job failed for some reason
with self.assertRaises(tenacity.RetryError):
self.get_hook.run_with_advanced_retry(
endpoint='v1/test',
_retry_args=retry_args
)
self.assertEqual(
self.get_hook._retry_obj.stop.max_attempt_number + 1,
mocked_session.call_count
)
self.counter = 0
self.count = count
self.kwargs = kwargs
def __call__(self):
"""
Raise an Forbidden until after count threshold has been crossed.
Then return True.
"""
if self.counter < self.count:
self.counter += 1
raise Forbidden(**self.kwargs)
return True
@hook.GoogleCloudBaseHook.quota_retry(wait=tenacity.wait_none())
def _retryable_test_with_temporary_quota_retry(thing):
return thing()
class QuotaRetryTestCase(unittest.TestCase): # ptlint: disable=invalid-name
def test_do_nothing_on_non_error(self):
result = _retryable_test_with_temporary_quota_retry(lambda: 42)
self.assertTrue(result, 42)
def test_retry_on_exception(self):
message = "POST https://translation.googleapis.com/language/translate/v2: User Rate Limit Exceeded"
errors = [
{
'message': 'User Rate Limit Exceeded',
'domain': 'usageLimits',
class FacebookReader(Reader):
def __init__(
self,
app_id,
app_secret,
access_token,
object_id,
object_type,
level,
ad_insights,
breakdown,
action_breakdown,
field,
time_increment,
start_date,
end_date,
date_preset,
add_date_to_report,
):
# Authentication inputs
self.app_id = app_id
self.app_secret = app_secret
self.access_token = access_token
self.api = FacebookAdsApi.init(self.app_id, self.app_secret,
self.access_token)
# Level inputs
self.object_ids = object_id
self.object_type = object_type
self.level = level
# Report inputs
self.ad_insights = ad_insights
self.breakdowns = list(breakdown)
self.action_breakdowns = list(action_breakdown)
self.fields = list(field)
self._field_paths = [
re.split(r"[\]\[]+", f.strip("]")) for f in self.fields
]
self._api_fields = list(
{f[0]
for f in self._field_paths if f[0] not in self.breakdowns})
# Date inputs
self.time_increment = time_increment or False
self.start_date = start_date
self.end_date = end_date
self.date_preset = date_preset
self.add_date_to_report = add_date_to_report
# Validate inputs
self.validate_inputs()
check_date_range_definition_conformity(self.start_date, self.end_date,
self.date_preset)
def validate_inputs(self):
"""
Validate combination of input parameters (triggered in FacebookReader constructor).
"""
self.validate_object_type_and_level_combination()
self.validate_ad_insights_level()
self.validate_ad_insights_breakdowns()
self.validate_ad_insights_action_breakdowns()
self.validate_ad_management_inputs()
def validate_object_type_and_level_combination(self):
if (self.level != self.object_type) and (
self.level not in EDGE_MAPPING[self.object_type]):
raise ClickException(
f"Wrong query. Asked level ({self.level}) is not compatible with object type ({self.object_type}).\
Please choose level from: {[self.object_type] + EDGE_MAPPING[self.object_type]}"
)
def validate_ad_insights_level(self):
if self.ad_insights:
if self.level == "creative" or self.object_type == "creative":
raise ClickException(
f"Wrong query. The 'creative' level is not available in Ad Insights queries.\
Accepted levels: {FACEBOOK_OBJECTS[1:]}")
def validate_ad_insights_breakdowns(self):
if self.ad_insights:
missing_breakdowns = {
f[0]
for f in self._field_paths
if (f[0] in BREAKDOWNS) and (f[0] not in self.breakdowns)
}
if missing_breakdowns != set():
raise ClickException(
f"Wrong query. Please add to Breakdowns: {missing_breakdowns}"
)
def validate_ad_insights_action_breakdowns(self):
if self.ad_insights:
missing_action_breakdowns = {
flt
for f in self._field_paths
for flt in get_action_breakdown_filters(f)
if flt not in self.action_breakdowns
}
if missing_action_breakdowns != set():
raise ClickException(
f"Wrong query. Please add to Action Breakdowns: {missing_action_breakdowns}"
)
def validate_ad_management_inputs(self):
if not self.ad_insights:
if self.breakdowns != [] or self.action_breakdowns != []:
raise ClickException(
"Wrong query. Ad Management queries do not accept Breakdowns nor Action Breakdowns."
)
if self.time_increment:
raise ClickException(
"Wrong query. Ad Management queries do not accept the time_increment parameter."
)
def get_params(self):
"""
Build the request parameters that will be sent to the API:
- If Ad Insights query: breakdown, action_breakdowns, level, time_range and date_preset
- If Ad Management query at the campaign, adset or ad level: time_range and date_preset
"""
params = {}
if self.ad_insights:
params["breakdowns"] = self.breakdowns
params["action_breakdowns"] = self.action_breakdowns
params["level"] = self.level
self.add_period_to_params(params)
else:
if self.level in ["campaign", "adset", "ad"]:
self.add_period_to_params(params)
return params
def add_period_to_params(self, params):
"""
Add the time_increment, time_range and/or date_preset keys to parameters.
- time_increment: available in Ad Insights queries
- time_range and date_preset: available in Ad Insights queries,
and in Ad Management queries at the campaign, adset or ad levels only
"""
if self.ad_insights and self.time_increment:
params["time_increment"] = self.time_increment
if self.ad_insights or self.level in ["campaign", "adset", "ad"]:
if self.start_date and self.end_date:
logger.info(
"Date format used for request: start_date and end_date")
params["time_range"] = self.create_time_range()
elif self.date_preset:
logger.info("Date format used for request: date_preset")
params["date_preset"] = self.date_preset
else:
logging.warning(
"No date range provided - Last 30 days by default")
logging.warning(
"https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights#parameters"
)
logger.warning(
"No date range provided - Last 30 days by default")
logger.warning(
"https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights#parameters"
)
def create_time_range(self):
return {
"since": self.start_date.strftime(DATEFORMAT),
"until": self.end_date.strftime(DATEFORMAT)
}
def create_object(self, object_id):
"""
Create a Facebook object based on the provided object_type and object_id.
"""
if self.object_type == "account":
object_id = "act_" + object_id
obj = OBJECT_CREATION_MAPPING[self.object_type](object_id)
return obj
def query_ad_insights(self, fields, params, object_id):
"""
Ad Insights documentation:
https://developers.facebook.com/docs/marketing-api/insights
"""
logger.info(
f"Running Facebook Ad Insights query on {self.object_type}_id: {object_id}"
)
# Step 1 - Create Facebook object
obj = self.create_object(object_id)
# Step 2 - Run Ad Insights query on Facebook object
report_job = self._get_report(obj, fields, params)
yield from report_job.get_result()
@retry(wait=wait_none(), stop=stop_after_attempt(3))
def _get_report(self, obj, fields, params):
async_job = obj.get_insights(fields=fields,
params=params,
is_async=True)
self._wait_for_100_percent_completion(async_job)
self._wait_for_complete_report(async_job)
return async_job
@retry(wait=wait_exponential(multiplier=5, max=300),
stop=stop_after_delay(2400))
def _wait_for_100_percent_completion(self, async_job):
async_job.api_get()
percent_completion = async_job[
AdReportRun.Field.async_percent_completion]
status = async_job[AdReportRun.Field.async_status]
logger.info(f"{status}: {percent_completion}%")
if status == "Job Failed":
logger.info(status)
elif percent_completion < 100:
raise Exception(f"{status}: {percent_completion}")
@retry(wait=wait_exponential(multiplier=10, max=60),
stop=stop_after_delay(300))
def _wait_for_complete_report(self, async_job):
async_job.api_get()
status = async_job[AdReportRun.Field.async_status]
if status == "Job Running":
raise Exception(status)
logger.info(status)
def query_ad_management(self, fields, params, object_id):
"""
Ad Management documentation:
https://developers.facebook.com/docs/marketing-api/reference/
Supported object nodes: AdAccount, Campaign, AdSet, Ad and AdCreative
"""
logger.info(
f"Running Ad Management query on {self.object_type}_id: {object_id}"
)
# Step 1 - Create Facebook object
obj = self.create_object(object_id)
# Step 2 - Run Ad Management query on the Facebook object itself,
# or on one of its edges (depending on the specified level)
if self.level == self.object_type:
yield obj.api_get(fields=fields, params=params)
else:
edge_objs = EDGE_QUERY_MAPPING[self.level](obj)
yield from self.get_edge_objs_records(edge_objs, fields, params)
def get_edge_objs_records(self, edge_objs, fields, params):
"""
Make batch Ad Management requests on a set of edge objects
(edge_objs being a facebook_business.api.Cursor object).
"""
total_edge_objs = edge_objs._total_count
total_batches = ceil(total_edge_objs / BATCH_SIZE_LIMIT)
logger.info(
f"Making {total_batches} batch requests on a total of {total_edge_objs} {self.level}s"
)
for batch in generate_batches(edge_objs, BATCH_SIZE_LIMIT):
# Create batch
api_batch = self.api.new_batch()
batch_responses = []
# Add each campaign request to batch
for obj in batch:
def callback_success(response):
batch_responses.append(response.json())
monitor_usage(response)
def callback_failure(response):
raise response.error()
obj.api_get(fields=fields,
params=params,
batch=api_batch,
success=callback_success,
failure=callback_failure)
# Execute batch
api_batch.execute()
yield from batch_responses
def format_and_yield(self, record):
"""
Parse a single record into an {item: value} dictionary.
"""
report = {}
for field_path in self._field_paths:
field_values = get_field_values(record,
field_path,
self.action_breakdowns,
visited=[])
if field_values:
report.update(field_values)
if self.add_date_to_report:
report["date"] = datetime.today().strftime(DATEFORMAT)
yield report
def result_generator(self, data):
"""
Parse all records into an {item: value} dictionary.
"""
for record in data:
yield from self.format_and_yield(record)
def get_data_for_object(self, object_id):
"""
Run an API query (Ad Insights or Ad Management) on a single object_id.
"""
params = self.get_params()
if self.ad_insights:
data = self.query_ad_insights(self._api_fields, params, object_id)
else:
data = self.query_ad_management(self._api_fields, params,
object_id)
yield from self.result_generator(data)
def get_data(self):
"""
Run API queries on all object_ids.
"""
for object_id in self.object_ids:
yield from self.get_data_for_object(object_id)
def read(self):
yield JSONStream(
"results_" + self.object_type + "_" + "_".join(self.object_ids),
self.get_data())
def test_it_raises_exception_when_timeout_reached(self, *_):
await_no_resources_found.retry.wait = wait_none()
await_no_resources_found.retry.stop = stop_after_attempt(1)
with pytest.raises(RetryError):
delete_namespace("a_namespace")
