def test_valid_random_range(self, rr): rr.return_value = 0 maximum = 100 get_exponential_backoff_interval(factor=40, retries=10, maximum=maximum, full_jitter=True) rr.assert_called_once_with(maximum + 1)
def test_without_jitter(self): assert get_exponential_backoff_interval( factor=4, retries=3, maximum=100, full_jitter=False ) == 4 * (2 ** 3)
def run(*args, **kwargs): try: return task._orig_run(*args, **kwargs) except Ignore: # If Ignore signal occurs task shouldn't be retried, # even if it suits autoretry_for list raise except Retry: raise except dont_autoretry_for: raise except autoretry_for as exc: if retry_backoff: retry_kwargs['countdown'] = \ get_exponential_backoff_interval( factor=retry_backoff, retries=task.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter) # Override max_retries if hasattr(task, 'override_max_retries'): retry_kwargs['max_retries'] = getattr( task, 'override_max_retries', task.max_retries) ret = task.retry(exc=exc, **retry_kwargs) # Stop propagation if hasattr(task, 'override_max_retries'): delattr(task, 'override_max_retries') raise ret
def store_result(self, task_id, result, state, traceback=None, request=None, **kwargs): """Update task state and result. if always_retry_backend_operation is activated, in the event of a recoverable exception, then retry operation with an exponential backoff until a limit has been reached. """ result = self.encode_result(result, state) retries = 0 while True: try: self._store_result(task_id, result, state, traceback, request=request, **kwargs) return result except Exception as exc: if self.always_retry and self.exception_safe_to_retry(exc): if retries < self.max_retries: retries += 1 # get_exponential_backoff_interval computes integers # and time.sleep accept floats for sub second sleep sleep_amount = get_exponential_backoff_interval( self.base_sleep_between_retries_ms, retries, self.max_sleep_between_retries_ms, True) / 1000 self._sleep(sleep_amount) else: raise_with_context( BackendStoreError("failed to store result on the backend", task_id=task_id, state=state), ) else: raise
def run(*args, **kwargs): try: return self._orig_run(*args, **kwargs) except self.autoretry_for as exc: if self.request_stack: if 'countdown' not in self.retry_kwargs: countdown = get_exponential_backoff_interval( factor=self.retry_backoff, retries=self.request.retries, maximum=self.retry_backoff_max, full_jitter=self.retry_jitter, ) retry_kwargs = self.retry_kwargs.copy() retry_kwargs.update({'countdown': countdown}) else: retry_kwargs = self.retry_kwargs retry_kwargs.update({'exc': exc}) raise self.retry(**retry_kwargs) else: logging.warning('no celery task request stack') logging.warning(traceback.format_exc()) finally: self.start_time = self.now_time
def test_bound_by_maximum(self): maximum_boundary = 100 assert get_exponential_backoff_interval( factor=40, retries=3, maximum=maximum_boundary ) == maximum_boundary
def run(*args, **kwargs): try: return task._orig_run(*args, **kwargs) except autoretry_for as exc: if retry_backoff: retry_kwargs['countdown'] = \ get_exponential_backoff_interval( factor=retry_backoff, retries=task.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter) raise task.retry(exc=exc, **retry_kwargs)
def run(*args, **kwargs): try: return task._orig_run(*args, **kwargs) except autoretry_for as exc: if retry_backoff: retry_kwargs['countdown'] = \ get_exponential_backoff_interval( factor=retry_backoff, retries=task.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter) raise task.retry(exc=exc, **retry_kwargs)
def run(*args, **kwargs): try: return task._orig_run(*args, **kwargs) except Ignore: # If Ignore signal occures task shouldn't be retried, # even if it suits autoretry_for list raise except autoretry_for as exc: if retry_backoff: retry_kwargs['countdown'] = \ get_exponential_backoff_interval( factor=retry_backoff, retries=task.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter) raise task.retry(exc=exc, **retry_kwargs)
def exponential_backoff(retries): """ Return a number of seconds to delay the next task attempt using an exponential back-off algorithm with jitter. See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ :param retries: Number of retries so far :return: number of seconds to delay the next try """ backoff_minutes = get_exponential_backoff_interval( factor=2, retries=retries, maximum=settings.CELERY_MAX_RETRY_DELAY_MINUTES, full_jitter=True, ) # wait for a minimum of 1 minute return max(1, backoff_minutes) * 60
def _get_retry_countdown(self, task_func): retry_backoff = int( self.task_kwargs.get('retry_backoff', True) ) retry_backoff_max = int( self.task_kwargs.get('retry_backoff_max', 600) ) retry_jitter = self.task_kwargs.get( 'retry_jitter', True ) countdown = get_exponential_backoff_interval( factor=retry_backoff, retries=task_func.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter ) return countdown
def get_task_meta(self, task_id, cache=True): """Get task meta from backend. if always_retry_backend_operation is activated, in the event of a recoverable exception, then retry operation with an exponential backoff until a limit has been reached. """ self._ensure_not_eager() if cache: try: return self._cache[task_id] except KeyError: pass retries = 0 while True: try: meta = self._get_task_meta_for(task_id) break except Exception as exc: if self.always_retry and self.exception_safe_to_retry(exc): if retries < self.max_retries: retries += 1 # get_exponential_backoff_interval computes integers # and time.sleep accept floats for sub second sleep sleep_amount = (get_exponential_backoff_interval( self.base_sleep_between_retries_ms, retries, self.max_sleep_between_retries_ms, True, ) / 1000) self._sleep(sleep_amount) else: raise_with_context( BackendGetMetaError("failed to get meta", task_id=task_id), ) else: raise if cache and meta.get("status") == states.SUCCESS: self._cache[task_id] = meta return meta
def prepare_models(self, engine): if not self.prepared: # SQLAlchemy will check if the items exist before trying to # create them, which is a race condition. If it raises an error # in one iteration, the next may pass all the existence checks # and the call will succeed. retries = 0 while True: try: ResultModelBase.metadata.create_all(engine) except DatabaseError: if retries < PREPARE_MODELS_MAX_RETRIES: sleep_amount_ms = get_exponential_backoff_interval( 10, retries, 1000, True) time.sleep(sleep_amount_ms / 1000) retries += 1 else: raise else: break self.prepared = True
def calculate_max_retries(retry_max_elapsed_backoff, retry_backoff_max): """ Calculate retry count to allow before reaching the max elapsed time. Args: retry_max_elapsed_backoff (int): maximum time in seconds to allow for waiting cumulatively across all retries retry_backoff_max (int): maximum time in seconds to allow for a retry Returns: int: number of allowed retries """ max_retries = 0 elapsed_time = 0 while elapsed_time < retry_max_elapsed_backoff: retry_factor = get_exponential_backoff_interval( factor=1, retries=max_retries, maximum=retry_backoff_max) elapsed_time += retry_factor if (elapsed_time < retry_max_elapsed_backoff): max_retries += 1 return max_retries
def wrapper(*args, **kwargs): task = args[0] assert isinstance(task, Task), 'bind=True must be set to enable retries' assert task.acks_late, 'acks_late=True must be set to send rejected tasks to the dead letter queue' autoretry_for = tuple( getattr(task, 'autoretry_for', ()) ) retry_backoff = int( getattr(task, 'retry_backoff', False) ) retry_backoff_max = int( getattr(task, 'retry_backoff_max', 600) ) retry_jitter = getattr(task, 'retry_jitter', True) countdown = None if retry_backoff: countdown = get_exponential_backoff_interval( factor=retry_backoff, retries=task.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter) try: if not autoretry_for: return func(*args, **kwargs) else: try: return func(*args, **kwargs) except autoretry_for as retry_exc: raise task.retry(exc=retry_exc, countdown=countdown) except TaskPredicate: # pass through celery specific exceptions raise except Exception as exc: # reject if max_retries exceeded raise Reject(exc, requeue=False) from exc
def test_negative_values(self): assert get_exponential_backoff_interval(factor=-40, retries=3, maximum=100) == 0
def test_negative_values(self): assert get_exponential_backoff_interval( factor=-40, retries=3, maximum=100 ) == 0