class RetryStrategies: __instance = None # A default strategy with the following attributes: # # - 3 attempts # - Exponential back off of (2 ^ retries) seconds # - Random jitter between retries of 0-2 seconds # - Retry on timeouts, connection errors, internal server errors and throttles __default_retry_strategy = retrying.Retrying( stop_max_attempt_number=3, wait_exponential_multiplier=1000, wait_exponential_max=10000, wait_jitter_max=2000, retry_on_exception= retry_on_timeouts_connection_internal_server_and_throttles) # Allows us to store different retry strategies so that we don't have to create Retry objects # all the time __availabile_strategies = { DEFAULT_RETRY_STRATEGY_NAME: __default_retry_strategy } def __new__(cls): if RetryStrategies.__instance is None: RetryStrategies.__instance = object.__new__(cls) return RetryStrategies.__instance def add_retry_strategy(self, strategy_name, retrying_obj): if not strategy_name or not retrying_obj: raise RuntimeError( 'A strategy name and retry strategy object must be provided') if strategy_name == DEFAULT_RETRY_STRATEGY_NAME: raise RuntimeError('You cannot overwrite the default strategy') self.__availabile_strategies[strategy_name] = retrying_obj def get_retry_strategy(self, strategy_name): return self.__availabile_strategies.get(strategy_name, None) def get_default_retry_strategy(self): return self.__default_retry_strategy
def db_setup_with_retry(db_name, db_host, db_port, username=None, password=None, ensure_indexes=True, ssl=False, ssl_keyfile=None, ssl_certfile=None, ssl_cert_reqs=None, ssl_ca_certs=None, ssl_match_hostname=True): """ This method is a retry version of db_setup. """ # Using as an annotation would be nice but annotations are evaluated at import # time and simple ways to use the annotation means the config gets read before # it is setup. Likely there is a way to use some proxies to delay the actual # reading of config values however this is lesser code. retrying_obj = retrying.Retrying( retry_on_exception=_retry_if_connection_error, wait_exponential_multiplier=cfg.CONF.database. connection_retry_backoff_mul * 1000, wait_exponential_max=cfg.CONF.database.connection_retry_backoff_max_s * 1000, stop_max_delay=cfg.CONF.database.connection_retry_max_delay_m * 60 * 1000) return retrying_obj.call(db_setup, db_name, db_host, db_port, username=username, password=password, ensure_indexes=ensure_indexes, ssl=ssl, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, ssl_cert_reqs=ssl_cert_reqs, ssl_ca_certs=ssl_ca_certs, ssl_match_hostname=ssl_match_hostname)
def acquire(self): """Acquire the Lock as configured""" opts = { k: v for k, v in self.options.items() if k in { 'stop_max_delay', 'stop_max_attempt_number', 'wait_exponential_max', 'wait_exponential_multiplier', 'wait_fixed', 'wait_random_max', 'wait_random_min', 'retry_on_exception', 'wrap_exception', } } with self.acquire_timer: success = retrying.Retrying(**opts).call(self._acquire_or_release) for p in self._all_fulfilled_iter(): self.reporter_class(**p.tags).lock_success( self.acquire_timer.duration) self.release_timer.start() return success
def wrapper(*args, **kwargs): job_id = get_named_arg('job_id', func, args, kwargs) logging.debug('args: %s -- kwargs: %s', args, kwargs) job = models.Job.query.get(job_id) logger = get_instance_logger(job.instance, task_id=job_id) task = args[func.func_code.co_varnames.index('self')] try: lock = redis.lock('tyr.lock|' + job.instance.name, timeout=self.timeout) locked = lock.acquire(blocking=False) except ConnectionError: logging.exception( 'Exception with redis while locking. Retrying in 10sec') task.retry(countdown=10, max_retries=10) if not locked: countdown = 300 logger.info('lock on %s retry %s in %s sec', job.instance.name, func.__name__, countdown) task.retry(countdown=countdown, max_retries=10) else: try: logger.debug('lock acquired on %s for %s', job.instance.name, func.__name__) return func(*args, **kwargs) finally: logger.debug('release lock on %s for %s', job.instance.name, func.__name__) # sometimes we are disconnected from redis when we want to release the lock, # so we retry only the release try: retrying.Retrying(stop_max_attempt_number=5, wait_fixed=1000).call( lock_release, lock, logger) except ValueError: # LockError(ValueError) since redis 3.0 logger.exception( "impossible to release lock: continue but following task may be locked :(" )
def wrapper(*args, **kwargs): hg = _Hourglass(log_periodicity) r = retrying.Retrying(wait_fixed=delay_ms, retry_on_exception=retry_on_exception, stop_func=functools.partial(never_stop, hg)) return r.call(func, *args, **kwargs)
def _wrapper(*args, **kwargs): r = retrying.Retrying(retry_on_exception=_retry_on_exception, wait_func=_backoff_sleep, stop_func=_print_stop) return r.call(f, *args, **kwargs)
def setup_class(cls): cls.krakens_pool = {} logging.info("Initing the tests {}, let's pop the krakens".format( cls.__name__)) cls.launch_all_krakens() instances_config_files = cls.create_dummy_json() i_manager.configuration_files = instances_config_files i_manager.initialisation() cls.mocks = [] for name in cls.krakens_pool: priority = cls.data_sets[name].get('priority', 0) logging.info('instance %s has priority %s', name, priority) is_free = cls.data_sets[name].get('is_free', False) is_open_data = cls.data_sets[name].get('is_open_data', False) scenario = cls.data_sets[name].get('scenario', 'default') cls.mocks.append( mock.patch.object(i_manager.instances[name], 'get_models', return_value=FakeModel( priority, is_free, is_open_data, scenario))) for m in cls.mocks: m.start() # we check that all instances are up for name in cls.krakens_pool: instance = i_manager.instances[name] try: retrying.Retrying(stop_max_delay=5000, wait_fixed=10, retry_on_result=lambda x: not instance. is_initialized).call(instance.init) except RetryError: logging.exception('impossible to start kraken {}'.format(name)) assert False, 'impossible to start a kraken' #we block the stat manager not to send anything to rabbit mq def mock_publish(self, stat): pass #we don't want to initialize rabbit for test. def mock_init(): pass StatManager.publish_request = mock_publish StatManager._init_rabbitmq = mock_init #we don't want to have anything to do with the jormun database either class bob: @classmethod def mock_get_token(cls, token, valid_until): #note, since get_from_token is a class method, we need to wrap it. #change that with a real mock framework pass User.get_from_token = bob.mock_get_token @property def mock_journey_order(self): return 'arrival_time' Instance.journey_order = mock_journey_order
def setup_class(cls): cls.tester = app.test_client() cls.krakens_pool = {} cls.uid = uuid.uuid1() logging.info("Initing the tests {}, let's pop the krakens".format( cls.__name__)) cls.global_jormun_setup() cls.launch_all_krakens() instances_config_files = cls.create_dummy_json() i_manager.configuration_files = instances_config_files i_manager.initialisation() cls.mocks = [] for name in cls.krakens_pool: priority = cls.data_sets[name].get('priority', 0) logging.info('instance %s has priority %s', name, priority) is_free = cls.data_sets[name].get('is_free', False) is_open_data = cls.data_sets[name].get('is_open_data', False) scenario = cls.data_sets[name].get('scenario', 'new_default') poi_dataset = cls.data_sets[name].get('poi_dataset', None) cls.mocks.append( mock.patch.object( i_manager.instances[name], 'get_models', return_value=FakeModel(priority, is_free, is_open_data, scenario, poi_dataset=poi_dataset), )) for m in cls.mocks: m.start() # we check that all instances are up for name in cls.krakens_pool: instance = i_manager.instances[name] try: retrying.Retrying( stop_max_attempt_number=5, wait_fixed=10, retry_on_result=lambda x: not instance.is_initialized, ).call(instance.init) except RetryError: logging.exception('impossible to start kraken {}'.format(name)) assert False, 'impossible to start a kraken' # we don't want to have anything to do with the jormun database either class bob: @classmethod def mock_get_token(cls, token, valid_until): # note, since get_from_token is a class method, we need to wrap it. # change that with a real mock framework pass User.get_from_token = bob.mock_get_token @property def mock_journey_order(self): return 'arrival_time' Instance.journey_order = mock_journey_order
def register_exchanges_with_retry(): retrying_obj = retrying.Retrying( retry_on_exception=socket.error, wait_fixed=cfg.CONF.messaging.connection_retry_wait, stop_max_attempt_number=cfg.CONF.messaging.connection_retries) return retrying_obj.call(register_exchanges)