def test_upload_file_raises_error_on_retry_exception(self, mock_upload_file): mock_upload_file.side_effect = exceptions.RetryError('Message', 'Cause') with self.assertRaises(cloud_storage.Error): self.cloud_storage_obj.upload_file(self.source_file_path, self.bucket_name, self.destination_blob_path)
def retry_wrapper(*args, **kwargs): sleep_generator = core_retry.exponential_sleep_generator( _DEFAULT_INITIAL_DELAY, _DEFAULT_MAXIMUM_DELAY, _DEFAULT_DELAY_MULTIPLIER, ) for sleep_time in itertools.islice(sleep_generator, retries + 1): try: result = callback(*args, **kwargs) if isinstance(result, tasklets.Future): result = yield result except Exception as e: # `e` is removed from locals at end of block error = e # See: https://goo.gl/5J8BMK if not is_transient_error(error): raise error else: raise tasklets.Return(result) yield tasklets.sleep(sleep_time) raise core_exceptions.RetryError( "Maximum number of {} retries exceeded while calling {}".format( retries, callback), cause=error, )
def retry_target(target, predicate, sleep_generator, deadline, on_error=None): """Call a function and retry if it fails. This is the lowest-level retry helper. Generally, you'll use the higher-level retry helper :class:`Retry`. Args: target(Callable): The function to call and retry. This must be a nullary function - apply arguments with `functools.partial`. predicate (Callable[Exception]): A callable used to determine if an exception raised by the target should be considered retryable. It should return True to retry or False otherwise. sleep_generator (Iterable[float]): An infinite iterator that determines how long to sleep between retries. deadline (float): How long to keep retrying the target. on_error (Callable): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. Returns: Any: the return value of the target function. Raises: google.api_core.RetryError: If the deadline is exceeded while retrying. ValueError: If the sleep generator stops yielding values. Exception: If the target raises a method that isn't retryable. """ if deadline is not None: deadline_datetime = (datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline)) else: deadline_datetime = None last_exc = None for sleep in sleep_generator: try: return target() # pylint: disable=broad-except # This function explicitly must deal with broad exceptions. except Exception as exc: if not predicate(exc): raise last_exc = exc if on_error is not None: on_error(exc) now = datetime_helpers.utcnow() if deadline_datetime is not None and deadline_datetime < now: six.raise_from( exceptions.RetryError( 'Deadline of {:.1f}s exceeded while calling {}'.format( deadline, target), last_exc), last_exc) _LOGGER.debug('Retrying due to {}, sleeping {:.1f}s ...'.format( last_exc, sleep)) time.sleep(sleep) raise ValueError('Sleep generator stopped yielding sleep values.')
def __call__(self, *args, **kwargs): self._state += 1 if self._state % 3 == 0: raise gexc.RetryError("", "") if self._state % 3 == 1: raise gexc.DeadlineExceeded("") if self._state % 3 == 2: return self.pull_response
def retry_wrapper(*args, **kwargs): from google.cloud.ndb import context as context_module sleep_generator = core_retry.exponential_sleep_generator( _DEFAULT_INITIAL_DELAY, _DEFAULT_MAXIMUM_DELAY, _DEFAULT_DELAY_MULTIPLIER, ) for sleep_time in itertools.islice(sleep_generator, retries + 1): context = context_module.get_context() if not context.in_retry(): # We need to be able to identify if we are inside a nested # retry. Here, we set the retry state in the context. This is # used for deciding if an exception should be raised # immediately or passed up to the outer retry block. context.set_retry_state(repr(callback)) try: result = callback(*args, **kwargs) if isinstance(result, tasklets.Future): result = yield result except exceptions.NestedRetryException as e: error = e except Exception as e: # `e` is removed from locals at end of block error = e # See: https://goo.gl/5J8BMK if not is_transient_error(error): # If we are in an inner retry block, use special nested # retry exception to bubble up to outer retry. Else, raise # actual exception. if context.get_retry_state() != repr(callback): message = getattr(error, "message", str(error)) raise exceptions.NestedRetryException(message) else: raise error else: raise tasklets.Return(result) finally: # No matter what, if we are exiting the top level retry, # clear the retry state in the context. if context.get_retry_state() == repr( callback): # pragma: NO BRANCH context.clear_retry_state() yield tasklets.sleep(sleep_time) raise core_exceptions.RetryError( "Maximum number of {} retries exceeded while calling {}".format( retries, callback), cause=error, )
def test_send_unary_retry_error(caplog): caplog.set_level(logging.DEBUG) manager, _, _, _, _, _ = make_running_manager() error = exceptions.RetryError("Too long a transient error", cause=Exception("Out of time!")) manager._client.acknowledge.side_effect = error with pytest.raises(exceptions.RetryError): manager.send( gapic_types.StreamingPullRequest(ack_ids=["ack_id1", "ack_id2"])) assert "RetryError while sending unary RPC" in caplog.text assert "signaled streaming pull manager shutdown" in caplog.text
class WaitForCompletionOperatorTest(unittest.TestCase): def setUp(self): super(WaitForCompletionOperatorTest, self).setUp() dag = models.DAG(dag_id=_DAG_ID, start_date=datetime.datetime.now()) self._task = wait_for_completion_operator.WaitForCompletionOperator( project_id=_PROJECT_ID, queue_location=_QUEUE_LOCATION, queue_name=_QUEUE_NAME, service_account_path=_SERVICE_ACCOUNT_PATH, try_count_limit=_TRY_COUNT_LIMIT, dag=dag, task_id=_TASK_ID) self._context = mock.MagicMock() self._mock_tasks_client = mock.patch( 'google.cloud.tasks.CloudTasksClient.from_service_account_json', autospec=True).start() self._mock_time_sleep = mock.patch('time.sleep').start() self.addCleanup(mock.patch.stopall) @parameterized.expand([([['dummy_task_0', 'dummy_task_1', 'dummy_task_2'], ['dummy_task_0'], []], 2), ([[]], 0)]) def test_execute_should_wait_until_queue_becomes_empty( self, sequence_of_tasks, expected_sleep_count): self._mock_tasks_client.return_value.list_tasks.side_effect = sequence_of_tasks self._task.execute(self._context) self.assertEqual(expected_sleep_count, self._mock_time_sleep.call_count) @parameterized.expand([[exceptions.GoogleAPICallError('')], [exceptions.RetryError('', Exception())], [ValueError()]]) def test_execute_when_exception_raised(self, api_exception): self._mock_tasks_client.return_value.list_tasks.side_effect = api_exception with self.assertRaises(airflow.AirflowException): self._task.execute(self._context) def test_execute_should_raise_exception_when_try_count_exceeds_limit(self): self._mock_tasks_client.return_value.list_tasks.return_value = [ 'blocked_task' ] with self.assertRaises(airflow.AirflowException): self._task.execute(self._context) self.assertEqual( _TRY_COUNT_LIMIT, self._mock_tasks_client.return_value.list_tasks.call_count) self.assertEqual(_TRY_COUNT_LIMIT, self._mock_time_sleep.call_count)
def test_subscribe_retry_error(mock_sub_future): mock_sub_future.subscriber_client.subscribe.side_effect = exceptions.RetryError('message', 'cause') mock_sub_future.set_subscription('test') result = mock_sub_future.subscribe(callback) assert result is False assert mock_sub_future.errors
def test_publish_message_retry_error(mock_publisher): mock_publisher.publish_client.publish.side_effect = exceptions.RetryError('message', 'cause') mock_publisher.set_topic('test') result = mock_publisher.publish_message('test') assert result is False assert mock_publisher.errors
def test_create_subscription_retry_error(mock_subscriber_configured): mock_subscriber_configured.subscriber_client.create_subscription.side_effect = exceptions.RetryError('message', 'cause') result = mock_subscriber_configured.create_subscription() assert result is False assert mock_subscriber_configured.errors
async def retry_target(target, predicate, sleep_generator, deadline, on_error=None): """Call a function and retry if it fails. This is the lowest-level retry helper. Generally, you'll use the higher-level retry helper :class:`Retry`. Args: target(Callable): The function to call and retry. This must be a nullary function - apply arguments with `functools.partial`. predicate (Callable[Exception]): A callable used to determine if an exception raised by the target should be considered retryable. It should return True to retry or False otherwise. sleep_generator (Iterable[float]): An infinite iterator that determines how long to sleep between retries. deadline (float): How long to keep retrying the target. The last sleep period is shortened as necessary, so that the last retry runs at ``deadline`` (and not considerably beyond it). on_error (Callable[Exception]): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. Returns: Any: the return value of the target function. Raises: google.api_core.RetryError: If the deadline is exceeded while retrying. ValueError: If the sleep generator stops yielding values. Exception: If the target raises a method that isn't retryable. """ deadline_dt = ((datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline)) if deadline else None) last_exc = None for sleep in sleep_generator: try: if not deadline_dt: return await target() else: return await asyncio.wait_for( target(), timeout=(deadline_dt - datetime_helpers.utcnow()).total_seconds(), ) # pylint: disable=broad-except # This function explicitly must deal with broad exceptions. except Exception as exc: if not predicate(exc) and not isinstance(exc, asyncio.TimeoutError): raise last_exc = exc if on_error is not None: on_error(exc) now = datetime_helpers.utcnow() if deadline_dt: if deadline_dt <= now: # Chains the raising RetryError with the root cause error, # which helps observability and debugability. raise exceptions.RetryError( "Deadline of {:.1f}s exceeded while calling {}".format( deadline, target), last_exc, ) from last_exc else: time_to_deadline = (deadline_dt - now).total_seconds() sleep = min(time_to_deadline, sleep) _LOGGER.debug("Retrying due to {}, sleeping {:.1f}s ...".format( last_exc, sleep)) await asyncio.sleep(sleep) raise ValueError("Sleep generator stopped yielding sleep values.")