class RetryStrategies:
__instance = None
# A default strategy with the following attributes:
#
# - 3 attempts
# - Exponential back off of (2 ^ retries) seconds
# - Random jitter between retries of 0-2 seconds
# - Retry on timeouts, connection errors, internal server errors and throttles
__default_retry_strategy = retrying.Retrying(
stop_max_attempt_number=3,
wait_exponential_multiplier=1000,
wait_exponential_max=10000,
wait_jitter_max=2000,
retry_on_exception=
retry_on_timeouts_connection_internal_server_and_throttles)
# Allows us to store different retry strategies so that we don't have to create Retry objects
# all the time
__availabile_strategies = {
DEFAULT_RETRY_STRATEGY_NAME: __default_retry_strategy
}
def __new__(cls):
if RetryStrategies.__instance is None:
RetryStrategies.__instance = object.__new__(cls)
return RetryStrategies.__instance
def add_retry_strategy(self, strategy_name, retrying_obj):
if not strategy_name or not retrying_obj:
raise RuntimeError(
'A strategy name and retry strategy object must be provided')
if strategy_name == DEFAULT_RETRY_STRATEGY_NAME:
raise RuntimeError('You cannot overwrite the default strategy')
self.__availabile_strategies[strategy_name] = retrying_obj
def get_retry_strategy(self, strategy_name):
return self.__availabile_strategies.get(strategy_name, None)
def get_default_retry_strategy(self):
return self.__default_retry_strategy
def db_setup_with_retry(db_name,
db_host,
db_port,
username=None,
password=None,
ensure_indexes=True,
ssl=False,
ssl_keyfile=None,
ssl_certfile=None,
ssl_cert_reqs=None,
ssl_ca_certs=None,
ssl_match_hostname=True):
"""
This method is a retry version of db_setup.
"""
# Using as an annotation would be nice but annotations are evaluated at import
# time and simple ways to use the annotation means the config gets read before
# it is setup. Likely there is a way to use some proxies to delay the actual
# reading of config values however this is lesser code.
retrying_obj = retrying.Retrying(
retry_on_exception=_retry_if_connection_error,
wait_exponential_multiplier=cfg.CONF.database.
connection_retry_backoff_mul * 1000,
wait_exponential_max=cfg.CONF.database.connection_retry_backoff_max_s *
1000,
stop_max_delay=cfg.CONF.database.connection_retry_max_delay_m * 60 *
1000)
return retrying_obj.call(db_setup,
db_name,
db_host,
db_port,
username=username,
password=password,
ensure_indexes=ensure_indexes,
ssl=ssl,
ssl_keyfile=ssl_keyfile,
ssl_certfile=ssl_certfile,
ssl_cert_reqs=ssl_cert_reqs,
ssl_ca_certs=ssl_ca_certs,
ssl_match_hostname=ssl_match_hostname)
def acquire(self):
"""Acquire the Lock as configured"""
opts = {
k: v
for k, v in self.options.items() if k in {
'stop_max_delay',
'stop_max_attempt_number',
'wait_exponential_max',
'wait_exponential_multiplier',
'wait_fixed',
'wait_random_max',
'wait_random_min',
'retry_on_exception',
'wrap_exception',
}
}
with self.acquire_timer:
success = retrying.Retrying(**opts).call(self._acquire_or_release)
for p in self._all_fulfilled_iter():
self.reporter_class(**p.tags).lock_success(
self.acquire_timer.duration)
self.release_timer.start()
return success
def wrapper(*args, **kwargs):
job_id = get_named_arg('job_id', func, args, kwargs)
logging.debug('args: %s -- kwargs: %s', args, kwargs)
job = models.Job.query.get(job_id)
logger = get_instance_logger(job.instance, task_id=job_id)
task = args[func.func_code.co_varnames.index('self')]
try:
lock = redis.lock('tyr.lock|' + job.instance.name,
timeout=self.timeout)
locked = lock.acquire(blocking=False)
except ConnectionError:
logging.exception(
'Exception with redis while locking. Retrying in 10sec')
task.retry(countdown=10, max_retries=10)
if not locked:
countdown = 300
logger.info('lock on %s retry %s in %s sec', job.instance.name,
func.__name__, countdown)
task.retry(countdown=countdown, max_retries=10)
else:
try:
logger.debug('lock acquired on %s for %s',
job.instance.name, func.__name__)
return func(*args, **kwargs)
finally:
logger.debug('release lock on %s for %s',
job.instance.name, func.__name__)
# sometimes we are disconnected from redis when we want to release the lock,
# so we retry only the release
try:
retrying.Retrying(stop_max_attempt_number=5,
wait_fixed=1000).call(
lock_release, lock, logger)
except ValueError: # LockError(ValueError) since redis 3.0
logger.exception(
"impossible to release lock: continue but following task may be locked :("
)
def wrapper(*args, **kwargs):
hg = _Hourglass(log_periodicity)
r = retrying.Retrying(wait_fixed=delay_ms,
retry_on_exception=retry_on_exception,
stop_func=functools.partial(never_stop, hg))
return r.call(func, *args, **kwargs)
def _wrapper(*args, **kwargs):
r = retrying.Retrying(retry_on_exception=_retry_on_exception,
wait_func=_backoff_sleep,
stop_func=_print_stop)
return r.call(f, *args, **kwargs)
def setup_class(cls):
cls.krakens_pool = {}
logging.info("Initing the tests {}, let's pop the krakens".format(
cls.__name__))
cls.launch_all_krakens()
instances_config_files = cls.create_dummy_json()
i_manager.configuration_files = instances_config_files
i_manager.initialisation()
cls.mocks = []
for name in cls.krakens_pool:
priority = cls.data_sets[name].get('priority', 0)
logging.info('instance %s has priority %s', name, priority)
is_free = cls.data_sets[name].get('is_free', False)
is_open_data = cls.data_sets[name].get('is_open_data', False)
scenario = cls.data_sets[name].get('scenario', 'default')
cls.mocks.append(
mock.patch.object(i_manager.instances[name],
'get_models',
return_value=FakeModel(
priority, is_free, is_open_data,
scenario)))
for m in cls.mocks:
m.start()
# we check that all instances are up
for name in cls.krakens_pool:
instance = i_manager.instances[name]
try:
retrying.Retrying(stop_max_delay=5000,
wait_fixed=10,
retry_on_result=lambda x: not instance.
is_initialized).call(instance.init)
except RetryError:
logging.exception('impossible to start kraken {}'.format(name))
assert False, 'impossible to start a kraken'
#we block the stat manager not to send anything to rabbit mq
def mock_publish(self, stat):
pass
#we don't want to initialize rabbit for test.
def mock_init():
pass
StatManager.publish_request = mock_publish
StatManager._init_rabbitmq = mock_init
#we don't want to have anything to do with the jormun database either
class bob:
@classmethod
def mock_get_token(cls, token, valid_until):
#note, since get_from_token is a class method, we need to wrap it.
#change that with a real mock framework
pass
User.get_from_token = bob.mock_get_token
@property
def mock_journey_order(self):
return 'arrival_time'
Instance.journey_order = mock_journey_order
def setup_class(cls):
cls.tester = app.test_client()
cls.krakens_pool = {}
cls.uid = uuid.uuid1()
logging.info("Initing the tests {}, let's pop the krakens".format(
cls.__name__))
cls.global_jormun_setup()
cls.launch_all_krakens()
instances_config_files = cls.create_dummy_json()
i_manager.configuration_files = instances_config_files
i_manager.initialisation()
cls.mocks = []
for name in cls.krakens_pool:
priority = cls.data_sets[name].get('priority', 0)
logging.info('instance %s has priority %s', name, priority)
is_free = cls.data_sets[name].get('is_free', False)
is_open_data = cls.data_sets[name].get('is_open_data', False)
scenario = cls.data_sets[name].get('scenario', 'new_default')
poi_dataset = cls.data_sets[name].get('poi_dataset', None)
cls.mocks.append(
mock.patch.object(
i_manager.instances[name],
'get_models',
return_value=FakeModel(priority,
is_free,
is_open_data,
scenario,
poi_dataset=poi_dataset),
))
for m in cls.mocks:
m.start()
# we check that all instances are up
for name in cls.krakens_pool:
instance = i_manager.instances[name]
try:
retrying.Retrying(
stop_max_attempt_number=5,
wait_fixed=10,
retry_on_result=lambda x: not instance.is_initialized,
).call(instance.init)
except RetryError:
logging.exception('impossible to start kraken {}'.format(name))
assert False, 'impossible to start a kraken'
# we don't want to have anything to do with the jormun database either
class bob:
@classmethod
def mock_get_token(cls, token, valid_until):
# note, since get_from_token is a class method, we need to wrap it.
# change that with a real mock framework
pass
User.get_from_token = bob.mock_get_token
@property
def mock_journey_order(self):
return 'arrival_time'
Instance.journey_order = mock_journey_order
def register_exchanges_with_retry():
retrying_obj = retrying.Retrying(
retry_on_exception=socket.error,
wait_fixed=cfg.CONF.messaging.connection_retry_wait,
stop_max_attempt_number=cfg.CONF.messaging.connection_retries)
return retrying_obj.call(register_exchanges)
