def delete(key: str) -> bool:
try:
if redisController.redis_obj.delete(key) != 0:
return False
except redis.RedisError as e:
raise redis.RedisError(f"failed to delete {key}. {e}")
return True
def query(self, *args, **kwargs):
retry_total = 0
retry_every = 0
while retry_total < self.proxy.retry_total:
if retry_every >= self.proxy.retry_every:
retry_every = 0
if self.proxy.switch == 'next':
self.proxy.pool_order = self.proxy.pool_order + 1 % self.proxy.host_cnt
else:
candidate_host = list(range(self.proxy.host_cnt))
candidate_host.remove(self.proxy.pool_order)
self.proxy.pool_order = random.choice(candidate_host)
if self.proxy.start == 'last':
self.proxy.make_pool(keep_last_pool=True,
order=self.proxy.pool_order)
else:
self.proxy.make_pool(order=self.proxy.pool_order)
try:
# print('trying ', self.proxy.config[self.proxy.pool_order % self.proxy.host_cnt])
# print('times(this host) ', retry_every, self.proxy.retry_every)
# print('times(all)', retry_total, self.proxy.retry_total)
return self.pika_exec(self.proxy.pool, *args, **kwargs)
except redis.TimeoutError:
retry_every += 1
except redis.ConnectionError:
retry_every += 1
retry_total += 1
raise redis.RedisError('all hosts have gone')
def query(self, *args, **kwargs):
retry_total = 0
retry_every = 0
while retry_total < self.proxy.retry_total:
if retry_every >= self.proxy.retry_every:
retry_every = 0
self.proxy.pool_order = self.proxy.pool_order + 1 % self.proxy.host_cnt
self.proxy.make_pool(order=self.proxy.pool_order)
try:
return self.pika_exec(self.proxy.pool, *args, **kwargs)
except redis.TimeoutError:
retry_every += 1
except redis.ConnectionError:
retry_every += 1
retry_total += 1
raise redis.RedisError('all hosts have gone')
def test_redis_list_emit_error():
logger = logging.getLogger()
handler = RedisListHandler(
key='test', max_messages=2, db=1)
handler.redis_client.pipeline = Mock(side_effect=redis.RedisError())
logger.addHandler(handler)
logger.warning('test')
def test_cleanup_thread_resilience(self, *args, **kwargs):
# test cleanup thread resilience to Redis Exceptions
redis_conn = MagicMock()
tc = ThreadedCounter(start_time=1442671176, cycle_time=1,
keep_max=1) # Initialize the counter
tc.setup(
redis_conn=redis_conn
) # The thread ready but not started since _threaded_start is mock patched
# Check that a non redis exception kills the thread (assurance that the correct method is tested)
tc.expire = MagicMock(side_effect=Exception("1"))
try:
tc._do_thread_work()
# this should NOT be reached
self.assertTrue(False)
except Exception as e:
# this should be reached
self.assertEqual(str(e), "1")
# Check that a redis exception doesn't kill the thread
tc.expire = MagicMock(side_effect=redis.RedisError())
try:
tc._do_thread_work()
# this should be reached
self.assertTrue(True)
except:
# this should NOT be reached
self.assertTrue(False)
async def test_store_should_propagate_redis_error_to_caller(self):
cache = redis_cache.RedisCache(REDIS_HOST, REDIS_PORT)
self.mock_redis.setex.side_effect = redis.RedisError()
with self.assertRaises(redis.RedisError):
await cache.add_cache_value(ODS_CODE, INTERACTION_ID,
VALUE_DICTIONARY)
def zadd(self, name, *args, **kwargs):
"""
Set any number of score, element-name pairs to the key ``name``. Pairs
can be specified in two ways:
As *args, in the form of: score1, name1, score2, name2, ...
or as **kwargs, in the form of: name1=score1, name2=score2, ...
The following example would add four values to the 'my-key' key:
redis.zadd('my-key', 1.1, 'name1', 2.2, 'name2', name3=3.3, name4=4.4)
"""
if len(args) % 2 != 0:
raise redis.RedisError("ZADD requires an equal number of "
"values and scores")
zset = self._db.setdefault(name, {})
added = 0
for score, value in zip(*[args[i::2] for i in range(2)]):
if value not in zset:
added += 1
try:
zset[value] = float(score)
except ValueError:
raise redis.ResponseError("value is not a valid float")
for value, score in kwargs.items():
if value not in zset:
added += 1
try:
zset[value] = float(score)
except ValueError:
raise redis.ResponseError("value is not a valid float")
return added
def get(key: str) -> str:
value = ''
try:
value = redisController.redis_obj.get(name=key)
except redis.RedisError as e:
raise redis.RedisError(f"failed to set {key}. {e}")
return value.decode()
def zadd(self, name, *args, incr=False, **kwargs):
"""
NOTE: The order of arguments differs from that of the official ZADD
command. For backwards compatability, this method accepts arguments
in the form of name1, score1, name2, score2, while the official Redis
documents expects score1, name1, score2, name2.
If you're looking to use the standard syntax, consider using the
StrictRedis class. See the API Reference section of the docs for more
information.
Set any number of element-name, score pairs to the key ``name``. Pairs
can be specified in two ways:
As *args, in the form of: name1, score1, name2, score2, ...
or as **kwargs, in the form of: name1=score1, name2=score2, ...
The following example would add four values to the 'my-key' key:
redis.zadd('my-key', 'name1', 1.1, 'name2', 2.2, name3=3.3, name4=4.4)
"""
pieces = []
if args:
if len(args) % 2 != 0:
raise redis.RedisError("ZADD requires an equal number of "
"values and scores")
pieces.extend(reversed(args))
for pair in iteritems(kwargs):
pieces.append(pair[1])
pieces.append(pair[0])
if incr:
pieces = ['incr'] + pieces
return self.execute_command('ZADD', name, *pieces)
def test_urlextractorexception_returns_error(self):
self.mock_redis.get.side_effect = redis.RedisError()
request = self.get_mock_request(urls=self.sample_urls)
with self.assertRaises(HTTPException) as cm:
get_metadata(self.metadata_client, self.get_config(), request)
self.assertEqual(cm.exception.response.status_code, 500)
def shutdown():
''' from github.com/andymcurdy/redis-py '''
try:
cli.execute_command(shutdown_cmd)
except redis.ConnectionError:
# Expected here
return
raise redis.RedisError('SHUTDOWN seems to have failed')
def test_set_attempts_when_errors(self):
store = Store(host="localhost", port="6379", db='_not_exist_test_db_')
with patch('redis.Redis.set') as mock:
mock.side_effect = redis.RedisError('Test error')
try:
val = store.set('test_key', 'test_val')
except redis.RedisError:
pass
self.assertGreater(mock.call_count, 1)
def mset(self, *args, **kwargs):
if args:
if len(args) != 1 or not isinstance(args[0], dict):
raise redis.RedisError(
'MSET requires **kwargs or a single dict arg')
kwargs.update(args[0])
for key, val in iteritems(kwargs):
self.set(key, val)
return True
def zadd(self, name, value=None, score=None, **pairs):
"""
For each kwarg in ``pairs``, add that item and it's score to the
sorted set ``name``.
The ``value`` and ``score`` arguments are deprecated.
"""
if value is not None or score is not None:
if value is None or score is None:
raise redis.RedisError(
"Both 'value' and 'score' must be specified to ZADD")
warnings.warn(
DeprecationWarning(
"Passing 'value' and 'score' has been deprecated. "
"Please pass via kwargs instead."))
pairs = {str(value): score}
elif not pairs:
raise redis.RedisError("ZADD is missing kwargs param")
return super(FakeRedis, self).zadd(name, **pairs)
def _connect(self):
if self.ssh_client == None:
ssh = paramiko.SSHClient()
self.ssh_client = ssh
else:
ssh = self.ssh_client
if ssh.get_transport() == None or not ssh.get_transport().is_active():
# self.ssh_client, either first use of a shared instance or an instance
# we just created ourself (because no shared instance was passed into our
# __init__) is not yet connected... use the connection info specified to
# __init__ to connect
if isinstance(self.ssh_key, str):
try:
pk = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.ssh_key))
except Exception as err:
raise redis.RedisError(f"Error reading SSH private key file: {str(err)}")
elif isinstance(self.ssh_key, bytes):
try:
pk = paramiko.RSAKey.from_private_key_blob(self.ssh_key)
except Exception as err:
raise redis.RedisError(f"Error loading SSH private key blob: {str(err)}")
elif isinstance(self.ssh_key, paramiko.PKey):
pk = self.ssh_key
else:
raise redis.RedisError(f"Error, ssh_key must be either a str (path to id_rsa private key file), bytes (contents of id_rsa private key), or an instance of paramiko.PKey. Note that str and bytes expect RSA private key data, if you wish to use another cipher please load the key yourself and pass in the paramiko.PKey subclass instance.")
# at this point we should be ready to call connect
try:
ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy)
ssh.load_system_host_keys()
ssh.connect(pkey=pk, hostname=self.ssh_host, username=self.ssh_user) #key_filename=os.path.expanduser(self.ssh_keyfile))
except Exception as err:
raise redis.RedisError(f"Error opening SSH connection: {str(err)}")
try:
channel = ssh.get_transport().open_channel(dest_addr=(self.ssh_rhost, self.ssh_rport), src_addr=("", 0xfffe), kind="direct-tcpip")
except Exception as err:
raise redis.RedisError(f"Error opening SSH channel: {str(err)}")
self.ssh_channel = channel
return channel
def _push(msg, client, keyname=loglist):
"""
Push message to the redis list which the logstash used.
Not to auto connect the redis.
Can use 'with redisDB.redis_connection() as :' to push many datas,but connect the redis once.
:param msg: json string or json string's list
:return: Result of rpush(the item number in the list)
"""
if not isinstance(client, redis.StrictRedis):
raise redis.RedisError('Must use "with redisDB.connection(rediscfg)"')
return client.rpush(keyname, msg)
def sort(self,
name,
start=None,
num=None,
by=None,
get=None,
desc=False,
alpha=False,
store=None):
"""Sort and return the list, set or sorted set at ``name``.
``start`` and ``num`` allow for paging through the sorted data
``by`` allows using an external key to weight and sort the items.
Use an "*" to indicate where in the key the item value is located
``get`` allows for returning items from external keys rather than the
sorted data itself. Use an "*" to indicate where int he key
the item value is located
``desc`` allows for reversing the sort
``alpha`` allows for sorting lexicographically rather than numerically
``store`` allows for storing the result of the sort into
the key ``store``
"""
if (start is None and num is not None) or \
(start is not None and num is None):
raise redis.RedisError(
"RedisError: ``start`` and ``num`` must both be specified")
try:
data = self._db[name][:]
if by is not None:
# _sort_using_by_arg mutates data so we don't
# need need a return value.
self._sort_using_by_arg(data, by=by)
elif not alpha:
data.sort(key=self._strtod_key_func)
else:
data.sort()
if not (start is None and num is None):
data = data[start:start + num]
if desc:
data = list(reversed(data))
if store is not None:
self._db[store] = data
return len(data)
else:
return self._retrive_data_from_sort(data, get)
except KeyError:
return []
def zadd(self, name, items, nx=False, xx=False, ch=False, incr=False):
if nx and xx:
raise redis.RedisError("ZADD can't use both NX and XX modes")
pieces = []
if nx:
pieces.append('NX')
if xx:
pieces.append('XX')
if ch:
pieces.append('CH')
if incr:
pieces.append('INCR')
for k, v in items.items():
pieces.extend([v, k])
return self.execute_command('ZADD', name, *pieces)
def _zrangebyscore(self, name, min, max, start, num, withscores, reverse):
if (start is not None and num is None) or \
(num is not None and start is None):
raise redis.RedisError("``start`` and ``num`` must both "
"be specified")
all_items = self._db.get(name, {})
in_order = self._get_zelements_in_order(all_items, reverse=reverse)
matches = []
for item in in_order:
if min <= all_items[item] <= max:
matches.append(item)
if start is not None:
matches = matches[start:start + num]
if withscores:
return [(k, all_items[k]) for k in matches]
return matches
def zadd(self, name, value=None, score=None, **pairs):
"""
For each kwarg in ``pairs``, add that item and it's score to the
sorted set ``name``.
The ``value`` and ``score`` arguments are deprecated.
"""
if value is not None or score is not None:
if value is None or score is None:
raise redis.RedisError(
"Both 'value' and 'score' must be specified to ZADD")
warnings.warn(DeprecationWarning(
"Passing 'value' and 'score' has been deprecated. " \
"Please pass via kwargs instead."))
else:
value = pairs.keys()[0]
score = pairs.values()[0]
self._db.setdefault(name, {})[value] = score
def test_redis_emit_error(redis_logger):
handler = redis_logger.handlers[0]
with patch.object(handler.redis_client, 'publish') as mock_publish:
mock_publish.side_effect = redis.RedisError()
redis_logger.warning('test2')
def set(key: str, value: str):
try:
redisController.redis_obj.set(name=key, value=value)
except redis.RedisError as e:
raise redis.RedisError(f"failed to set {key}. {e}")
def expire(self, key, seconds):
if self.broken:
raise redis.RedisError()
self.callstack.append('expire')
return key == 'good'
def test_pocket_exception_returns_500(self):
self.mock_redis.get.side_effect = redis.RedisError()
response = self.client.get('/v2/recommendations')
self.assertEqual(response.status_code, 500)
def test_set_raises_when_disconnected(self):
store = Store(host="localhost", port="6379", db='_not_exist_test_db_')
with patch('redis.Redis.set') as mock:
mock.side_effect = redis.RedisError('Test error')
self.assertRaises(redis.RedisError, store.set, 'test_key',
'test_val')
def throw(*_, **__):
""" Throw an exception """
raise redis.RedisError("DB exception")
def execute(*args, **kwargs):
times[0] += 1
if times[0] < 4:
raise redis.RedisError('Err: connection timeout')
