def test_upload_file_raises_error_on_retry_exception(self, mock_upload_file):
mock_upload_file.side_effect = exceptions.RetryError('Message', 'Cause')
with self.assertRaises(cloud_storage.Error):
self.cloud_storage_obj.upload_file(self.source_file_path,
self.bucket_name,
self.destination_blob_path)
def retry_wrapper(*args, **kwargs):
sleep_generator = core_retry.exponential_sleep_generator(
_DEFAULT_INITIAL_DELAY,
_DEFAULT_MAXIMUM_DELAY,
_DEFAULT_DELAY_MULTIPLIER,
)
for sleep_time in itertools.islice(sleep_generator, retries + 1):
try:
result = callback(*args, **kwargs)
if isinstance(result, tasklets.Future):
result = yield result
except Exception as e:
# `e` is removed from locals at end of block
error = e # See: https://goo.gl/5J8BMK
if not is_transient_error(error):
raise error
else:
raise tasklets.Return(result)
yield tasklets.sleep(sleep_time)
raise core_exceptions.RetryError(
"Maximum number of {} retries exceeded while calling {}".format(
retries, callback),
cause=error,
)
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
"""Call a function and retry if it fails.
This is the lowest-level retry helper. Generally, you'll use the
higher-level retry helper :class:`Retry`.
Args:
target(Callable): The function to call and retry. This must be a
nullary function - apply arguments with `functools.partial`.
predicate (Callable[Exception]): A callable used to determine if an
exception raised by the target should be considered retryable.
It should return True to retry or False otherwise.
sleep_generator (Iterable[float]): An infinite iterator that determines
how long to sleep between retries.
deadline (float): How long to keep retrying the target.
on_error (Callable): A function to call while processing a retryable
exception. Any error raised by this function will *not* be
caught.
Returns:
Any: the return value of the target function.
Raises:
google.api_core.RetryError: If the deadline is exceeded while retrying.
ValueError: If the sleep generator stops yielding values.
Exception: If the target raises a method that isn't retryable.
"""
if deadline is not None:
deadline_datetime = (datetime_helpers.utcnow() +
datetime.timedelta(seconds=deadline))
else:
deadline_datetime = None
last_exc = None
for sleep in sleep_generator:
try:
return target()
# pylint: disable=broad-except
# This function explicitly must deal with broad exceptions.
except Exception as exc:
if not predicate(exc):
raise
last_exc = exc
if on_error is not None:
on_error(exc)
now = datetime_helpers.utcnow()
if deadline_datetime is not None and deadline_datetime < now:
six.raise_from(
exceptions.RetryError(
'Deadline of {:.1f}s exceeded while calling {}'.format(
deadline, target), last_exc), last_exc)
_LOGGER.debug('Retrying due to {}, sleeping {:.1f}s ...'.format(
last_exc, sleep))
time.sleep(sleep)
raise ValueError('Sleep generator stopped yielding sleep values.')
def __call__(self, *args, **kwargs):
self._state += 1
if self._state % 3 == 0:
raise gexc.RetryError("", "")
if self._state % 3 == 1:
raise gexc.DeadlineExceeded("")
if self._state % 3 == 2:
return self.pull_response
def retry_wrapper(*args, **kwargs):
from google.cloud.ndb import context as context_module
sleep_generator = core_retry.exponential_sleep_generator(
_DEFAULT_INITIAL_DELAY,
_DEFAULT_MAXIMUM_DELAY,
_DEFAULT_DELAY_MULTIPLIER,
)
for sleep_time in itertools.islice(sleep_generator, retries + 1):
context = context_module.get_context()
if not context.in_retry():
# We need to be able to identify if we are inside a nested
# retry. Here, we set the retry state in the context. This is
# used for deciding if an exception should be raised
# immediately or passed up to the outer retry block.
context.set_retry_state(repr(callback))
try:
result = callback(*args, **kwargs)
if isinstance(result, tasklets.Future):
result = yield result
except exceptions.NestedRetryException as e:
error = e
except Exception as e:
# `e` is removed from locals at end of block
error = e # See: https://goo.gl/5J8BMK
if not is_transient_error(error):
# If we are in an inner retry block, use special nested
# retry exception to bubble up to outer retry. Else, raise
# actual exception.
if context.get_retry_state() != repr(callback):
message = getattr(error, "message", str(error))
raise exceptions.NestedRetryException(message)
else:
raise error
else:
raise tasklets.Return(result)
finally:
# No matter what, if we are exiting the top level retry,
# clear the retry state in the context.
if context.get_retry_state() == repr(
callback): # pragma: NO BRANCH
context.clear_retry_state()
yield tasklets.sleep(sleep_time)
raise core_exceptions.RetryError(
"Maximum number of {} retries exceeded while calling {}".format(
retries, callback),
cause=error,
)
def test_send_unary_retry_error(caplog):
caplog.set_level(logging.DEBUG)
manager, _, _, _, _, _ = make_running_manager()
error = exceptions.RetryError("Too long a transient error",
cause=Exception("Out of time!"))
manager._client.acknowledge.side_effect = error
with pytest.raises(exceptions.RetryError):
manager.send(
gapic_types.StreamingPullRequest(ack_ids=["ack_id1", "ack_id2"]))
assert "RetryError while sending unary RPC" in caplog.text
assert "signaled streaming pull manager shutdown" in caplog.text
class WaitForCompletionOperatorTest(unittest.TestCase):
def setUp(self):
super(WaitForCompletionOperatorTest, self).setUp()
dag = models.DAG(dag_id=_DAG_ID, start_date=datetime.datetime.now())
self._task = wait_for_completion_operator.WaitForCompletionOperator(
project_id=_PROJECT_ID,
queue_location=_QUEUE_LOCATION,
queue_name=_QUEUE_NAME,
service_account_path=_SERVICE_ACCOUNT_PATH,
try_count_limit=_TRY_COUNT_LIMIT,
dag=dag,
task_id=_TASK_ID)
self._context = mock.MagicMock()
self._mock_tasks_client = mock.patch(
'google.cloud.tasks.CloudTasksClient.from_service_account_json',
autospec=True).start()
self._mock_time_sleep = mock.patch('time.sleep').start()
self.addCleanup(mock.patch.stopall)
@parameterized.expand([([['dummy_task_0', 'dummy_task_1', 'dummy_task_2'],
['dummy_task_0'], []], 2), ([[]], 0)])
def test_execute_should_wait_until_queue_becomes_empty(
self, sequence_of_tasks, expected_sleep_count):
self._mock_tasks_client.return_value.list_tasks.side_effect = sequence_of_tasks
self._task.execute(self._context)
self.assertEqual(expected_sleep_count,
self._mock_time_sleep.call_count)
@parameterized.expand([[exceptions.GoogleAPICallError('')],
[exceptions.RetryError('', Exception())],
[ValueError()]])
def test_execute_when_exception_raised(self, api_exception):
self._mock_tasks_client.return_value.list_tasks.side_effect = api_exception
with self.assertRaises(airflow.AirflowException):
self._task.execute(self._context)
def test_execute_should_raise_exception_when_try_count_exceeds_limit(self):
self._mock_tasks_client.return_value.list_tasks.return_value = [
'blocked_task'
]
with self.assertRaises(airflow.AirflowException):
self._task.execute(self._context)
self.assertEqual(
_TRY_COUNT_LIMIT,
self._mock_tasks_client.return_value.list_tasks.call_count)
self.assertEqual(_TRY_COUNT_LIMIT, self._mock_time_sleep.call_count)
def test_subscribe_retry_error(mock_sub_future):
mock_sub_future.subscriber_client.subscribe.side_effect = exceptions.RetryError('message', 'cause')
mock_sub_future.set_subscription('test')
result = mock_sub_future.subscribe(callback)
assert result is False
assert mock_sub_future.errors
def test_publish_message_retry_error(mock_publisher):
mock_publisher.publish_client.publish.side_effect = exceptions.RetryError('message', 'cause')
mock_publisher.set_topic('test')
result = mock_publisher.publish_message('test')
assert result is False
assert mock_publisher.errors
def test_create_subscription_retry_error(mock_subscriber_configured):
mock_subscriber_configured.subscriber_client.create_subscription.side_effect = exceptions.RetryError('message', 'cause')
result = mock_subscriber_configured.create_subscription()
assert result is False
assert mock_subscriber_configured.errors
async def retry_target(target,
predicate,
sleep_generator,
deadline,
on_error=None):
"""Call a function and retry if it fails.
This is the lowest-level retry helper. Generally, you'll use the
higher-level retry helper :class:`Retry`.
Args:
target(Callable): The function to call and retry. This must be a
nullary function - apply arguments with `functools.partial`.
predicate (Callable[Exception]): A callable used to determine if an
exception raised by the target should be considered retryable.
It should return True to retry or False otherwise.
sleep_generator (Iterable[float]): An infinite iterator that determines
how long to sleep between retries.
deadline (float): How long to keep retrying the target. The last sleep
period is shortened as necessary, so that the last retry runs at
``deadline`` (and not considerably beyond it).
on_error (Callable[Exception]): A function to call while processing a
retryable exception. Any error raised by this function will *not*
be caught.
Returns:
Any: the return value of the target function.
Raises:
google.api_core.RetryError: If the deadline is exceeded while retrying.
ValueError: If the sleep generator stops yielding values.
Exception: If the target raises a method that isn't retryable.
"""
deadline_dt = ((datetime_helpers.utcnow() +
datetime.timedelta(seconds=deadline))
if deadline else None)
last_exc = None
for sleep in sleep_generator:
try:
if not deadline_dt:
return await target()
else:
return await asyncio.wait_for(
target(),
timeout=(deadline_dt -
datetime_helpers.utcnow()).total_seconds(),
)
# pylint: disable=broad-except
# This function explicitly must deal with broad exceptions.
except Exception as exc:
if not predicate(exc) and not isinstance(exc,
asyncio.TimeoutError):
raise
last_exc = exc
if on_error is not None:
on_error(exc)
now = datetime_helpers.utcnow()
if deadline_dt:
if deadline_dt <= now:
# Chains the raising RetryError with the root cause error,
# which helps observability and debugability.
raise exceptions.RetryError(
"Deadline of {:.1f}s exceeded while calling {}".format(
deadline, target),
last_exc,
) from last_exc
else:
time_to_deadline = (deadline_dt - now).total_seconds()
sleep = min(time_to_deadline, sleep)
_LOGGER.debug("Retrying due to {}, sleeping {:.1f}s ...".format(
last_exc, sleep))
await asyncio.sleep(sleep)
raise ValueError("Sleep generator stopped yielding sleep values.")
