def run_in_thread(self, sleep_time=0):
for channel, handler in iteritems(self.channels):
if handler is None:
raise PubSubError("Channel: '%s' has no handler registered")
for pattern, handler in iteritems(self.patterns):
if handler is None:
raise PubSubError("Pattern: '%s' has no handler registered")
pubsub = self
class WorkerThread(threading.Thread):
def __init__(self, *args, **kwargs):
super(WorkerThread, self).__init__(*args, **kwargs)
self._running = False
def run(self):
if self._running:
return
self._running = True
while self._running and pubsub.subscribed:
pubsub.get_message(ignore_subscribe_messages=True)
mod_time.sleep(sleep_time)
def stop(self):
self._running = False
self.join()
thread = WorkerThread()
thread.start()
return thread
def _rc_msetnx(self, mapping):
"""
Sets each key in the ``mapping`` dict to its corresponding value if
none of the keys are already set
"""
for k, v in iteritems(mapping):
if self.exists(k):
return False
result = True
for k, v in iteritems(mapping):
result = result and self.set(k, v)
return result
def zadd(self, name, *args, **kwargs):
"""
NOTE: The order of arguments differs from that of the official ZADD
command. For backwards compatability, this method accepts arguments
in the form of name1, score1, name2, score2, while the official Redis
documents expects score1, name1, score2, name2.
If you're looking to use the standard syntax, consider using the
StrictRedis class. See the API Reference section of the docs for more
information.
Set any number of element-name, score pairs to the key ``name``. Pairs
can be specified in two ways:
As *args, in the form of: name1, score1, name2, score2, ...
or as **kwargs, in the form of: name1=score1, name2=score2, ...
The following example would add four values to the 'my-key' key:
redis.zadd('my-key', 'name1', 1.1, 'name2', 2.2, name3=3.3, name4=4.4)
"""
pieces = []
if args:
if len(args) % 2 != 0:
raise RedisError("ZADD requires an equal number of values and scores")
pieces.extend(reversed(args))
for pair in iteritems(kwargs):
pieces.append(pair[1])
pieces.append(pair[0])
return self.execute_command('ZADD', name, *pieces)
def test_binary_lists(self, r):
mapping = {
b('foo bar'): [b('1'), b('2'), b('3')],
b('foo\r\nbar\r\n'): [b('4'), b('5'), b('6')],
b('foo\tbar\x07'): [b('7'), b('8'), b('9')],
}
# fill in lists
for key, value in iteritems(mapping):
r.rpush(key, *value)
# check that KEYS returns all the keys as they are
assert sorted(r.keys('*')) == sorted(list(iterkeys(mapping)))
# check that it is possible to get list content by key name
for key, value in iteritems(mapping):
assert r.lrange(key, 0, -1) == value
def test_binary_lists(self, r):
mapping = {
b("foo bar"): [b("1"), b("2"), b("3")],
b("foo\r\nbar\r\n"): [b("4"), b("5"), b("6")],
b("foo\tbar\x07"): [b("7"), b("8"), b("9")],
}
# fill in lists
for key, value in iteritems(mapping):
r.rpush(key, *value)
# check that KEYS returns all the keys as they are
assert sorted(r.keys("*")) == sorted(list(iterkeys(mapping)))
# check that it is possible to get list content by key name
for key, value in iteritems(mapping):
assert r.lrange(key, 0, -1) == value
def test_msetnx_kwargs(self, r):
d = {"a": b("1"), "b": b("2"), "c": b("3")}
assert r.msetnx(**d)
d2 = {"a": b("x"), "d": b("4")}
assert not r.msetnx(**d2)
for k, v in iteritems(d):
assert r[k] == v
assert r.get("d") is None
def mset(self, *args, **kwargs):
if args:
if len(args) != 1 or not isinstance(args[0], dict):
raise RedisError('MSET requires **kwargs or a single dict arg')
kwargs.update(args[0])
mapping = {self.appendKeys(key): value for key, value in iteritems(kwargs)}
return self.redis.mset(mapping)
def test_msetnx_kwargs(self, r):
d = {'a': b('1'), 'b': b('2'), 'c': b('3')}
assert r.msetnx(**d)
d2 = {'a': b('x'), 'd': b('4')}
assert not r.msetnx(**d2)
for k, v in iteritems(d):
assert r[k] == v
assert r.get('d') is None
def on_connect(self, connection):
"Re-subscribe to any channels and patterns previously subscribed to"
# NOTE: for python3, we can't pass bytestrings as keyword arguments
# so we need to decode channel/pattern names back to unicode strings
# before passing them to [p]subscribe.
if self.channels:
channels = {}
for k, v in iteritems(self.channels):
if not self.decode_responses:
k = k.decode(self.encoding, self.encoding_errors)
channels[k] = v
self.subscribe(**channels)
if self.patterns:
patterns = {}
for k, v in iteritems(self.patterns):
if not self.decode_responses:
k = k.decode(self.encoding, self.encoding_errors)
patterns[k] = v
self.psubscribe(**patterns)
def hmset(self, name, mapping):
"""
Set key to value within hash ``name`` for each corresponding
key and value from the ``mapping`` dict.
"""
if not mapping:
raise DataError("'hmset' with 'mapping' of length 0")
items = []
for pair in iteritems(mapping):
items.extend(pair)
return self.execute_command('HMSET', name, *items)
def _rc_keys(self, pattern='*'):
"Returns a list of keys matching ``pattern``"
result = []
for alias, redisent in iteritems(self.redises):
if alias.find('_slave') == -1:
continue
result.extend(redisent.keys(pattern))
return result
def _rc_dbsize(self):
"Returns the number of keys in the current database"
result = 0
for alias, redisent in iteritems(self.redises):
if alias.find('_slave') == -1:
continue
result += redisent.dbsize()
return result
def from_url(cls, url, **kwargs):
"""
return a sentinel object from url
``url`` sentinels://<host1>:<port1>,<host2>:<port2>/<db>?<querystring>
``param`` kwargs: parameters for constructing Sentinel object
``return`` tuple of sentinel object, db from url, service_name in url
url example::
sentinels://10.1.2.122:17700
sentinels://10.1.2.122:17700, 10.1.2.128:17700
sentinels://node1:17700,node2:17700
if db appears in both querystring and path, use path as first choice.
For example:
sentinels://node1:17700,node2:17700/1?db=2 return db 2
"""
url_string = url
url = urlparse(url)
if url.scheme != "sentinels":
raise Exception("sentinel url must start with sentinels://")
# in python2.6, custom URL schemes don't recognize querystring values
# they're left as part of the url.path.
if '?' in url.path and not url.query:
# chop the querystring including the ? off the end of the url
# and reparse it.
qs = url.path.split('?', 1)[1]
url = urlparse(url_string[:-(len(qs) + 1)])
else:
qs = url.query
url_options = {} # redis options
for name, value in iteritems(parse_qs(qs)):
if value and len(value) > 0:
url_options[name] = value[0]
service_name = url_options.get('service_name')
sentinels = [host.split(':') for host in url.netloc.split(',')]
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
db_from_url = url_options.get('db')
if db_from_url is None and url.path:
try:
db_from_url = int(url.path.replace('/', ''))
except (AttributeError, ValueError):
pass
return Sentinel(sentinels, **kwargs), db_from_url, service_name
def mset(self, *args, **kwargs):
"""
Sets key/values based on a mapping. Mapping can be supplied as a single
dictionary argument or as kwargs.
"""
if args:
if len(args) != 1 or not isinstance(args[0], dict):
raise RedisError('MSET requires **kwargs or a single dict arg')
kwargs.update(args[0])
items = []
for pair in iteritems(kwargs):
items.extend(pair)
return self.execute_command('MSET', *items)
def __init__(self, cluster={}, db=0):
#raise exception when wrong server hash
if 'nodes' not in cluster or 'master_of' not in cluster:
raise Exception(
"rediscluster: Please set a correct array of redis cluster.")
self.cluster = cluster
self.no_servers = len(cluster['master_of'])
slaves = dictvalues(cluster['master_of'])
self.redises = {}
#connect to all servers
for alias, server in iteritems(cluster['nodes']):
try:
self.__redis = redis.StrictRedis(db=db, **server)
sla = self.__redis.config_get('slaveof')['slaveof']
if alias in slaves and sla == '':
raise redis.DataError(
"rediscluster: server %s is not a slave." % (server,))
except Exception as e:
#if node is slave and is down, replace its connection with its master's
try:
ms = [k for k, v in iteritems(cluster['master_of'])
if v == alias and (sla != '' or cluster['nodes'][k] == cluster['nodes'][v])][0]
except IndexError:
ms = None
if ms is not None:
try:
self.__redis = redis.StrictRedis(
db=db, **cluster['nodes'][ms])
self.__redis.info()
except Exception as e:
raise redis.ConnectionError("rediscluster cannot connect to: %s %s" % (cluster['nodes'][ms], e))
else:
raise redis.ConnectionError(
"rediscluster cannot connect to: %s %s" % (server, e))
self.redises[alias] = self.__redis
def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None,
**connection_kwargs):
# if sentinel_kwargs isn't defined, use the socket_* options from
# connection_kwargs
if sentinel_kwargs is None:
sentinel_kwargs = dict([(k, v)
for k, v in iteritems(connection_kwargs)
if k.startswith('socket_')
])
self.sentinel_kwargs = sentinel_kwargs
self.sentinels = [StrictRedis(hostname, port, **self.sentinel_kwargs)
for hostname, port in sentinels]
self.min_other_sentinels = min_other_sentinels
self.connection_kwargs = connection_kwargs
def mset(self, *args, **kwargs):
"""
Sets key/values based on a mapping. Mapping can be supplied as a single
dictionary argument or as kwargs.
Cluster impl: Itterate over all items and do SET on each (k,v) pair
"""
if args:
if len(args) != 1 or not isinstance(args[0], dict):
raise RedisError('MSET requires **kwargs or a single dict arg')
kwargs.update(args[0])
for pair in iteritems(kwargs):
self.set(pair[0], pair[1])
return True
def run_in_thread(self, sleep_time=0):
'''
Replacement for the default PubSub run_in_thread method from
http://github.com/andymccurdy/redis-py/master/redis/client.py
'''
for channel, handler in iteritems(self.channels):
if handler is None:
raise PubSubError("Channel: '%s' has no handler registered" % channel)
for pattern, handler in iteritems(self.channels):
if handler is None:
raise PubSubError("Pattern: '%s' has no handler registered" % pattern)
pubsub = self
class WorkerThread(threading.Thread):
''' Listens for messages on subscriptions '''
def __init__(self, *args, **kwargs):
super(WorkerThread, self).__init__(*args, **kwargs)
self.daemon = True
self._running = False
def run(self):
''' loop while running on subscriptions '''
if self._running:
return
self._running = True
while self._running and pubsub.subscribed:
pubsub.get_message(ignore_subscribe_messages=True)
mod_time.sleep(sleep_time)
def stop(self):
''' stops the main loop '''
self._running = False
self.join()
thread = WorkerThread()
thread.start()
return thread
def msetnx(self, *args, **kwargs):
"""
Sets key/values based on a mapping if none of the keys are already set.
Mapping can be supplied as a single dictionary argument or as kwargs.
Returns a boolean indicating if the operation was successful.
"""
if args:
if len(args) != 1 or not isinstance(args[0], dict):
raise RedisError('MSETNX requires **kwargs or a single '
'dict arg')
kwargs.update(args[0])
items = []
for pair in iteritems(kwargs):
items.extend(pair)
return self.execute_command('MSETNX', *items)
def mset(self, *args, **kwargs):
"""
Sets key/values based on a mapping. Mapping can be supplied as a single
dictionary argument or as kwargs.
Cluster impl:
Itterate over all items and do SET on each (k,v) pair
Operation is no longer atomic.
"""
if args:
if len(args) != 1 or not isinstance(args[0], dict):
raise RedisError('MSET requires **kwargs or a single dict arg')
kwargs.update(args[0])
for pair in iteritems(kwargs):
self.set(pair[0], pair[1])
return True
def __init__(self,
sentinels,
min_other_sentinels=0,
sentinel_kwargs=None,
**connection_kwargs):
# if sentinel_kwargs isn't defined, use the socket_* options from
# connection_kwargs
if sentinel_kwargs is None:
sentinel_kwargs = dict([(k, v)
for k, v in iteritems(connection_kwargs)
if k.startswith('socket_')])
self.sentinel_kwargs = sentinel_kwargs
self.sentinels = [
StrictRedis(hostname, port, **self.sentinel_kwargs)
for hostname, port in sentinels
]
self.min_other_sentinels = min_other_sentinels
self.connection_kwargs = connection_kwargs
def function(*args, **kwargs):
if name not in StrictRedisCluster._loop_keys:
if name in StrictRedisCluster._tag_keys and not isinstance(args[0], list):
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError("rediscluster: Command %s Not Supported (each key name has its own node)" % name)
#get the hash key depending on tags or not
hkey = args[0]
#take care of tagged key names for forcing multiple keys on the same node, e.g. r.set(['userinfo', "age:uid"], value)
if isinstance(args[0], list):
hkey = args[0][0]
L = list(args)
L[0] = args[0][1]
args = tuple(L)
#get the node number
node = self._getnodenamefor(hkey)
redisent = self.redises[self.cluster['default_node']]
if name in StrictRedisCluster._write_keys:
redisent = self.redises[node]
elif name in StrictRedisCluster._read_keys:
redisent = self.redises[
self.cluster['master_of'][node]]
#Execute the command on the server
return getattr(redisent, name)(*args, **kwargs)
else:
result = {}
for alias, redisent in iteritems(self.redises):
if name in StrictRedisCluster._write_keys and alias not in self.cluster['master_of']:
res = None
else:
res = getattr(redisent, name)(*args, **kwargs)
if name == 'keys':
result.append(res)
else:
result[alias] = res
return result
def subscribe(self, *args, **kwargs):
"""
Subscribe to channels. Channels supplied as keyword arguments expect
a channel name as the key and a callable as the value. A channel's
callable will be invoked automatically when a message is received on
that channel rather than producing a message via ``listen()`` or
``get_message()``.
"""
if args:
args = list_or_args(args[0], args[1:])
new_channels = {}
new_channels.update(dict.fromkeys(imap(self.encode, args)))
for channel, handler in iteritems(kwargs):
new_channels[self.encode(channel)] = handler
ret_val = self.execute_command('SUBSCRIBE', *iterkeys(new_channels))
# update the channels dict AFTER we send the command. we don't want to
# subscribe twice to these channels, once for the command and again
# for the reconnection.
self.channels.update(new_channels)
return ret_val
def _connect(self):
"Create a TCP socket connection"
# we want to mimic what socket.create_connection does to support
# ipv4/ipv6, but we want to set options prior to calling
# socket.connect()
err = None
for res in socket.getaddrinfo(self.host, self.port, self.socket_type,
socket.SOCK_STREAM):
family, socktype, proto, canonname, socket_address = res
sock = None
try:
sock = socket.socket(family, socktype, proto)
# TCP_NODELAY
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# TCP_KEEPALIVE
if self.socket_keepalive:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
for k, v in iteritems(self.socket_keepalive_options):
sock.setsockopt(socket.IPPROTO_TCP, k, v)
# set the socket_connect_timeout before we connect
sock.settimeout(self.socket_connect_timeout)
# connect
sock.connect(socket_address)
# set the socket_timeout now that we're connected
sock.settimeout(self.socket_timeout)
return sock
except socket.error as _:
err = _
if sock is not None:
sock.close()
if err is not None:
raise err
raise socket.error("socket.getaddrinfo returned an empty list")
def _connect(self):
"Create a TCP socket connection"
# we want to mimic what socket.create_connection does to support
# ipv4/ipv6, but we want to set options prior to calling
# socket.connect()
err = None
for res in socket.getaddrinfo(self.host, self.port, 0,
socket.SOCK_STREAM):
family, socktype, proto, canonname, socket_address = res
sock = None
try:
sock = socket.socket(family, socktype, proto)
# TCP_NODELAY
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# TCP_KEEPALIVE
if self.socket_keepalive:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
for k, v in iteritems(self.socket_keepalive_options):
sock.setsockopt(socket.SOL_TCP, k, v)
# set the socket_connect_timeout before we connect
sock.settimeout(self.socket_connect_timeout)
# connect
sock.connect(socket_address)
# set the socket_timeout now that we're connected
sock.settimeout(self.socket_timeout)
return sock
except socket.error as _:
err = _
if sock is not None:
sock.close()
if err is not None:
raise err
raise socket.error("socket.getaddrinfo returned an empty list")
def zaddnx(self, name, *args, **kwargs):
"""Like zadd but don't update the score of existing elements,
Set any number of score, element-name pairs to the key ``name``. Pairs
can be specified in two ways:
As *args, in the form of: score1, name1, score2, name2, ...
or as **kwargs, in the form of: name1=score1, name2=score2, ...
The following example would add four values to the 'my-key' key:
redis.zaddnx('my-key', 1.1, 'name1', 2.2, 'name2', name3=3.3, name4=4.4)
"""
pieces = []
if args:
if len(args) % 2 != 0:
raise RedisError("ZADDNX requires an equal number of "
"values and scores")
pieces.extend(args)
for pair in iteritems(kwargs):
pieces.append(pair[1])
pieces.append(pair[0])
return ZADDNX(keys=[name], args=pieces, client=self)
def zadd(self, name, *args, **kwargs):
"""
Set any number of score, element-name pairs to the key ``name``. Pairs
can be specified in two ways:
As *args, in the form of: score1, name1, score2, name2, ...
or as **kwargs, in the form of: name1=score1, name2=score2, ...
The following example would add four values to the 'my-key' key:
redis.zadd('my-key', 1.1, 'name1', 2.2, 'name2', name3=3.3, name4=4.4)
If using non-strict Redis (strict_redis=False), args are expected in swapped form:
redis.zadd('my-key', 'name1', 1.1, 'name2', 2.2, name3=3.3, name4=4.4)
"""
pieces = []
if args:
if len(args) % 2 != 0:
raise RedisError("ZADD requires an equal number of "
"values and scores")
pieces.extend(self.strict_redis and args or reversed(args))
for pair in iteritems(kwargs):
pieces.append(pair[1])
pieces.append(pair[0])
return self.execute_command('ZADD', name, *pieces)
def from_url(cls, url, db=None, decode_components=False, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
rediss://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
Three URL schemes are supported:
redis:// creates a normal TCP socket connection
rediss:// creates a SSL wrapped TCP socket connection
unix:// creates a Unix Domain Socket connection
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
The ``decode_components`` argument allows this function to work with
percent-encoded URLs. If this argument is set to ``True`` all ``%xx``
escapes will be replaced by their single-character equivalents after
the URL has been parsed. This only applies to the ``hostname``,
``path``, and ``password`` components.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. The querystring
arguments ``socket_connect_timeout`` and ``socket_timeout`` if supplied
are parsed as float values. The arguments ``socket_keepalive`` and
``retry_on_timeout`` are parsed to boolean values that accept
True/False, Yes/No values to indicate state. Invalid types cause a
``UserWarning`` to be raised. In the case of conflicting arguments,
querystring arguments always win.
"""
url_string = url
url = urlparse(url)
qs = ''
# in python2.6, custom URL schemes don't recognize querystring values
# they're left as part of the url.path.
if '?' in url.path and not url.query:
# chop the querystring including the ? off the end of the url
# and reparse it.
qs = url.path.split('?', 1)[1]
url = urlparse(url_string[:-(len(qs) + 1)])
else:
qs = url.query
url_options = {}
for name, value in iteritems(parse_qs(qs)):
if value and len(value) > 0:
parser = URL_QUERY_ARGUMENT_PARSERS.get(name)
if parser:
try:
url_options[name] = parser(value[0])
except (TypeError, ValueError):
warnings.warn(UserWarning(
"Invalid value for `%s` in connection URL." % name
))
else:
url_options[name] = value[0]
if decode_components:
password = unquote(url.password) if url.password else None
path = unquote(url.path) if url.path else None
hostname = unquote(url.hostname) if url.hostname else None
else:
password = url.password
path = url.path
hostname = url.hostname
# We only support redis:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': password,
'path': path,
'connection_class': UnixDomainSocketConnection,
})
else:
url_options.update({
'host': hostname,
'port': int(url.port or 6379),
'password': password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and path:
try:
url_options['db'] = int(path.replace('/', ''))
except (AttributeError, ValueError):
pass
if url.scheme == 'rediss':
url_options['connection_class'] = SSLConnection
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
# backwards compatability
if 'charset' in kwargs:
warnings.warn(DeprecationWarning(
'"charset" is deprecated. Use "encoding" instead'))
kwargs['encoding'] = kwargs.pop('charset')
if 'errors' in kwargs:
warnings.warn(DeprecationWarning(
'"errors" is deprecated. Use "encoding_errors" instead'))
kwargs['encoding_errors'] = kwargs.pop('errors')
return cls(**kwargs)
def from_url(cls, url, db=None, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
rediss://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
Three URL schemes are supported:
redis:// creates a normal TCP socket connection
rediss:// creates a SSL wrapped TCP socket connection
unix:// creates a Unix Domain Socket connection
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
"""
url_string = url
url = urlparse(url)
qs = ''
# in python2.6, custom URL schemes don't recognize querystring values
# they're left as part of the url.path.
if '?' in url.path and not url.query:
# chop the querystring including the ? off the end of the url
# and reparse it.
qs = url.path.split('?', 1)[1]
url = urlparse(url_string[:-(len(qs) + 1)])
else:
qs = url.query
url_options = {}
for name, value in iteritems(parse_qs(qs)):
if value and len(value) > 0:
url_options[name] = value[0]
# We only support redis:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': url.password,
'path': url.path,
'connection_class': UnixDomainSocketConnection,
})
else:
url_options.update({
'host': url.hostname,
'port': int(url.port or 6379),
'password': url.password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and url.path:
try:
url_options['db'] = int(url.path.replace('/', ''))
except (AttributeError, ValueError):
pass
if url.scheme == 'rediss':
url_options['connection_class'] = SSLConnection
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
# backwards compatability
if 'charset' in kwargs:
warnings.warn(
DeprecationWarning(
'"charset" is deprecated. Use "encoding" instead'))
kwargs['encoding'] = kwargs.pop('charset')
if 'errors' in kwargs:
warnings.warn(
DeprecationWarning(
'"errors" is deprecated. Use "encoding_errors" instead'))
kwargs['encoding_errors'] = kwargs.pop('errors')
return cls(**kwargs)
def from_url(cls, url, db=None, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
rediss://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
Three URL schemes are supported:
redis:// creates a normal TCP socket connection
rediss:// creates a SSL wrapped TCP socket connection
unix:// creates a Unix Domain Socket connection
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
"""
url_string = url
url = urlparse(url)
qs = ''
# in python2.6, custom URL schemes don't recognize querystring values
# they're left as part of the url.path.
if '?' in url.path and not url.query:
# chop the querystring including the ? off the end of the url
# and reparse it.
qs = url.path.split('?', 1)[1]
url = urlparse(url_string[:-(len(qs) + 1)])
else:
qs = url.query
url_options = {}
for name, value in iteritems(parse_qs(qs)):
if value and len(value) > 0:
url_options[name] = value[0]
# We only support redis:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': url.password,
'path': url.path,
'connection_class': UnixDomainSocketConnection,
})
else:
url_options.update({
'host': url.hostname,
'port': int(url.port or 6379),
'password': url.password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and url.path:
try:
url_options['db'] = int(url.path.replace('/', ''))
except (AttributeError, ValueError):
pass
if url.scheme == 'rediss':
url_options['connection_class'] = SSLConnection
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
# backwards compatability
if 'charset' in kwargs:
warnings.warn(DeprecationWarning(
'"charset" is deprecated. Use "encoding" instead'))
kwargs['encoding'] = kwargs.pop('charset')
if 'errors' in kwargs:
warnings.warn(DeprecationWarning(
'"errors" is deprecated. Use "encoding_errors" instead'))
kwargs['encoding_errors'] = kwargs.pop('errors')
return cls(**kwargs)
def function(*args, **kwargs):
if name not in StrictRedisCluster._loop_keys:
# take care of hash tags
tag_start = None
key_type = hash_tag = ''
# since we don't have "first item" in dict,
# this list is needed in order to check hash_tag in mset({"a{a}": "a", "b":"b"})
list_ht = []
if isinstance(args[0], (basestring, bytes)):
key_type = 'string'
list_ht.append(args[0])
else:
if isinstance(args[0], list):
key_type = 'list'
list_ht.append(args[0][0])
else:
key_type = 'dict'
list_ht = iterkeys(args[0])
# check for hash tags
for k in list_ht:
try:
tag_start = k.index('{')
hash_tag = k
break
except Exception as e:
tag_start = None
# trigger error msg on tag keys unless we have hash tags e.g. "bar{zap}"
if name in StrictRedisCluster._tag_keys and not tag_start:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError("rediscluster: Command %s Not Supported (each key name has its own node)" % name)
# get the hash key
hkey = args[0]
# take care of hash tags names for forcing multiple keys on the same node,
# e.g. r.set("bar{zap}", "bar"), r.mget(["foo{foo}","bar"])
if tag_start is not None:
L = list(args)
if key_type != 'string':
if key_type == 'list':
hkey = L[0][0][tag_start + 1:-1]
L[0][0] = L[0][0][0:tag_start]
else:
hkey = hash_tag[tag_start + 1:-1]
L[0][hash_tag[0:tag_start]] = L[0][hash_tag]
del L[0][hash_tag]
else:
hkey = L[0][tag_start + 1:-1]
L[0] = L[0][0:tag_start]
args = tuple(L)
# get the node number
node = self.getnodenamefor(hkey)
if name in StrictRedisCluster._write_keys:
redisent = self.redises[node]
elif name in StrictRedisCluster._read_keys:
redisent = self.redises[node + '_slave']
else:
raise redis.DataError("rediscluster: Command %s Not Supported (each key name has its own node)" % name)
# Execute the command on the server
return getattr(redisent, name)(*args, **kwargs)
else:
# take care of keys that don't need to go through master and slaves redis servers
if name not in self._loop_keys_admin:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError("rediscluster: Command %s Not Supported (each key name has its own node)" % name)
result = {}
for alias, redisent in iteritems(self.redises):
if (name in StrictRedisCluster._write_keys and alias.find('_slave') >= 0) or (name in StrictRedisCluster._read_keys and alias.find('_slave') == -1):
res = None
else:
res = getattr(redisent, name)(*args, **kwargs)
result[alias] = res
return result
def __init__(self, cluster={}, db=0, mastersonly=False):
# raise exception when wrong server hash
if 'nodes' not in cluster:
raise Exception(
"rediscluster: Please set a correct array of redis cluster.")
self.cluster = cluster
have_master_of = 'master_of' in self.cluster
self.no_servers = len(self.cluster['master_of']) if have_master_of else len(self.cluster['nodes'])
self.redises = {}
redises_cons = {}
self.cluster['slaves'] = {}
# connect to all servers
for alias, server in iteritems(self.cluster['nodes']):
if have_master_of and alias not in self.cluster['master_of']:
continue
server_str = str(server)
if server_str in redises_cons:
self.redises[alias] = redises_cons[server_str]['master']
self.redises[alias +
'_slave'] = redises_cons[server_str]['slave']
self.cluster['slaves'][alias +
'_slave'] = redises_cons[server_str]['slave_node']
else:
try:
# connect to master
self.__redis = redis.StrictRedis(db=db, **server)
if not mastersonly and not have_master_of:
info = self.__redis.info()
if info['role'] != 'master':
raise redis.DataError(
"rediscluster: server %s is not a master." % (server,))
self.redises[alias] = self.__redis
redises_cons[server_str] = {}
redises_cons[server_str]['master'] = self.redises[alias]
# connect to slave
slave_connected = False
slave = {}
if not mastersonly:
if have_master_of:
slave = self.cluster[
'nodes'][self.cluster['master_of'][alias]]
elif 'connected_slaves' in info and info['connected_slaves'] > 0:
slave_host, slave_port, slave_online = info[
'slave0'].split(',')
if slave_online == 'online':
slave = {'host': slave_host, 'port': slave_port}
if slave :
try:
redis_slave = redis.StrictRedis(host=slave['host'], port=int(slave['port']), db=db)
self.redises[alias + '_slave'] = redis_slave
self.cluster['slaves'][alias + '_slave'] = {
'host': slave['host'], 'port': slave['port']}
redises_cons[server_str][
'slave'] = self.redises[alias + '_slave']
redises_cons[server_str]['slave_node'] = self.cluster['slaves'][alias + '_slave']
slave_connected = True
except redis.RedisError as e:
pass
# "RedisCluster cannot connect to: " + slave_host +':'+ slave_port
if not slave_connected:
self.redises[alias + '_slave'] = self.redises[alias]
self.cluster['slaves'][alias + '_slave'] = server
redises_cons[server_str][
'slave'] = self.redises[alias + '_slave']
redises_cons[server_str]['slave_node'] = self.cluster[
'slaves'][alias + '_slave']
except redis.RedisError as e:
raise redis.ConnectionError(
"rediscluster cannot connect to: %s %s" % (server, e))
def __init__(self, cluster={}, db=0, mastersonly=False):
# raise exception when wrong server hash
if 'nodes' not in cluster:
raise Exception(
"rediscluster: Please set a correct array of redis cluster.")
self.cluster = cluster
have_master_of = 'master_of' in self.cluster
self.no_servers = len(
self.cluster['master_of']) if have_master_of else len(
self.cluster['nodes'])
self.redises = {}
redises_cons = {}
self.cluster['slaves'] = {}
# connect to all servers
for alias, server in iteritems(self.cluster['nodes']):
if have_master_of and alias not in self.cluster['master_of']:
continue
server_str = str(server)
if server_str in redises_cons:
self.redises[alias] = redises_cons[server_str]['master']
self.redises[alias +
'_slave'] = redises_cons[server_str]['slave']
self.cluster['slaves'][
alias + '_slave'] = redises_cons[server_str]['slave_node']
else:
try:
# connect to master
self.__redis = redis.StrictRedis(db=db, **server)
if not mastersonly and not have_master_of:
info = self.__redis.info()
if info['role'] != 'master':
raise redis.DataError(
"rediscluster: server %s is not a master." %
(server, ))
self.redises[alias] = self.__redis
redises_cons[server_str] = {}
redises_cons[server_str]['master'] = self.redises[alias]
# connect to slave
slave_connected = False
slave = {}
if not mastersonly:
if have_master_of:
slave = self.cluster['nodes'][
self.cluster['master_of'][alias]]
elif 'connected_slaves' in info and info[
'connected_slaves'] > 0:
slave_host, slave_port, slave_online = info[
'slave0'].split(',')
if slave_online == 'online':
slave = {
'host': slave_host,
'port': slave_port
}
if slave:
try:
redis_slave = redis.StrictRedis(host=slave['host'],
port=int(
slave['port']),
db=db)
self.redises[alias + '_slave'] = redis_slave
self.cluster['slaves'][alias + '_slave'] = {
'host': slave['host'],
'port': slave['port']
}
redises_cons[server_str]['slave'] = self.redises[
alias + '_slave']
redises_cons[server_str][
'slave_node'] = self.cluster['slaves'][
alias + '_slave']
slave_connected = True
except redis.RedisError as e:
pass
# "RedisCluster cannot connect to: " + slave_host +':'+ slave_port
if not slave_connected:
self.redises[alias + '_slave'] = self.redises[alias]
self.cluster['slaves'][alias + '_slave'] = server
redises_cons[server_str]['slave'] = self.redises[
alias + '_slave']
redises_cons[server_str]['slave_node'] = self.cluster[
'slaves'][alias + '_slave']
except redis.RedisError as e:
raise redis.ConnectionError(
"rediscluster cannot connect to: %s %s" % (server, e))
def test_mset_kwargs(self, r):
d = {'a': b('1'), 'b': b('2'), 'c': b('3')}
assert r.mset(**d)
for k, v in iteritems(d):
assert r[k] == v
def function(*args, **kwargs):
if name not in StrictRedisCluster._loop_keys:
# take care of hash tags
tag_start = None
key_type = hash_tag = ''
# since we don't have "first item" in dict,
# this list is needed in order to check hash_tag in mset({"a{a}": "a", "b":"b"})
list_ht = []
if isinstance(args[0], (basestring, bytes)):
key_type = 'string'
list_ht.append(args[0])
else:
if isinstance(args[0], list):
key_type = 'list'
list_ht.append(args[0][0])
else:
key_type = 'dict'
list_ht = dictkeys(args[0])
# check for hash tags
for k in list_ht:
try:
tag_start = k.index('{')
hash_tag = k
break
except Exception as e:
tag_start = None
# trigger error msg on tag keys unless we have hash tags e.g. "bar{zap}"
if name in StrictRedisCluster._tag_keys and not tag_start:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError(
"rediscluster: Command %s Not Supported (each key name has its own node)"
% name)
# get the hash key
hkey = args[0]
# take care of hash tags names for forcing multiple keys on the same node,
# e.g. r.set("bar{zap}", "bar"), r.mget(["foo{foo}","bar"])
if tag_start is not None:
L = list(args)
if key_type != 'string':
if key_type == 'list':
hkey = L[0][0][tag_start + 1:-1]
L[0][0] = L[0][0][0:tag_start]
else:
hkey = hash_tag[tag_start + 1:-1]
L[0][hash_tag[0:tag_start]] = L[0][hash_tag]
del L[0][hash_tag]
else:
hkey = L[0][tag_start + 1:-1]
L[0] = L[0][0:tag_start]
args = tuple(L)
# get the node number
node = self._getnodenamefor(hkey)
if name in StrictRedisCluster._write_keys:
redisent = self.redises[node]
elif name in StrictRedisCluster._read_keys:
redisent = self.redises[node + '_slave']
else:
raise redis.DataError(
"rediscluster: Command %s Not Supported (each key name has its own node)"
% name)
# Execute the command on the server
return getattr(redisent, name)(*args, **kwargs)
else:
# take care of keys that don't need to go through master and slaves redis servers
if name not in self._loop_keys_admin:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError(
"rediscluster: Command %s Not Supported (each key name has its own node)"
% name)
result = {}
for alias, redisent in iteritems(self.redises):
if (name in StrictRedisCluster._write_keys
and alias.find('_slave') >= 0) or (
name in StrictRedisCluster._read_keys
and alias.find('_slave') == -1):
res = None
else:
res = getattr(redisent, name)(*args, **kwargs)
result[alias] = res
return result
def _rc_mset(self, mapping):
"Sets each key in the ``mapping`` dict to its corresponding value"
result = True
for k, v in iteritems(mapping):
result = result and self.set(k, v)
return result
def test_mset(self, r):
d = {'a': b'1', 'b': b'2', 'c': b'3'}
assert r.mset(d)
for k, v in iteritems(d):
assert r[k] == v
def from_url(cls, url, db=None, decode_components=False, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
rediss://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
Three URL schemes are supported:
- ```redis://``
<https://www.iana.org/assignments/uri-schemes/prov/redis>`_ creates a
normal TCP socket connection
- ```rediss://``
<https://www.iana.org/assignments/uri-schemes/prov/rediss>`_ creates
a SSL wrapped TCP socket connection
- ``unix://`` creates a Unix Domain Socket connection
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
The ``decode_components`` argument allows this function to work with
percent-encoded URLs. If this argument is set to ``True`` all ``%xx``
escapes will be replaced by their single-character equivalents after
the URL has been parsed. This only applies to the ``hostname``,
``path``, and ``password`` components.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. The querystring
arguments ``socket_connect_timeout`` and ``socket_timeout`` if supplied
are parsed as float values. The arguments ``socket_keepalive`` and
``retry_on_timeout`` are parsed to boolean values that accept
True/False, Yes/No values to indicate state. Invalid types cause a
``UserWarning`` to be raised. In the case of conflicting arguments,
querystring arguments always win.
"""
url = urlparse(url)
url_options = {}
for name, value in iteritems(parse_qs(url.query)):
if value and len(value) > 0:
parser = URL_QUERY_ARGUMENT_PARSERS.get(name)
if parser:
try:
url_options[name] = parser(value[0])
except (TypeError, ValueError):
warnings.warn(
UserWarning(
"Invalid value for `%s` in connection URL." %
name))
else:
url_options[name] = value[0]
if decode_components:
password = unquote(url.password) if url.password else None
path = unquote(url.path) if url.path else None
hostname = unquote(url.hostname) if url.hostname else None
else:
password = url.password
path = url.path
hostname = url.hostname
# We only support redis://, rediss:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': password,
'path': path,
'connection_class': UnixDomainSocketConnection,
})
elif url.scheme in ('redis', 'rediss'):
url_options.update({
'host': hostname,
'port': int(url.port or 6379),
'password': password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and path:
try:
url_options['db'] = int(path.replace('/', ''))
except (AttributeError, ValueError):
pass
if url.scheme == 'rediss':
url_options['connection_class'] = SSLConnection
else:
valid_schemes = ', '.join(('redis://', 'rediss://', 'unix://'))
raise ValueError('Redis URL must specify one of the following'
'schemes (%s)' % valid_schemes)
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
# backwards compatability
if 'charset' in kwargs:
warnings.warn(
DeprecationWarning(
'"charset" is deprecated. Use "encoding" instead'))
kwargs['encoding'] = kwargs.pop('charset')
if 'errors' in kwargs:
warnings.warn(
DeprecationWarning(
'"errors" is deprecated. Use "encoding_errors" instead'))
kwargs['encoding_errors'] = kwargs.pop('errors')
return cls(**kwargs)
def _connect(self):
"Create a TCP socket connection"
# we want to mimic what socket.create_connection does to support
# ipv4/ipv6, but we want to set options prior to calling
# socket.connect()
if self._iostream:
return
current = self._get_current_greenlet()
parent = current.parent
err = None
for res in socket.getaddrinfo(self.host, self.port, 0,
socket.SOCK_STREAM):
family, socktype, proto, _, socket_address = res
sock = None
try:
sock = socket.socket(family, socktype, proto)
# TCP_NODELAY
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# TCP_KEEPALIVE
if self.socket_keepalive:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
for k, v in iteritems(self.socket_keepalive_options):
sock.setsockopt(socket.SOL_TCP, k, v)
self._iostream = self._wrap_socket(sock)
timeout = self.socket_connect_timeout
if timeout:
self._timeout_handle = self._ioloop.add_timeout(
datetime.timedelta(seconds=timeout),
functools.partial(self._handle_timeout, current),
)
handle = generate_handle()
self._events[handle] = True
self._iostream.set_close_callback(
functools.partial(self._handle_error, current, handle)
)
try:
self._iostream.connect(
socket_address,
callback=functools.partial(self._handle_connect, current, handle)
)
# yield back to parent, wait for connect, error or timeout
parent.switch()
finally:
if self._timeout_handle:
self._ioloop.remove_timeout(self._timeout_handle)
self._timeout_handle = None
self._iostream.set_close_callback(None)
del self._events[handle]
return sock
except ConnectionError as _:
err = _
if sock is not None:
sock.close()
if self._iostream is not None:
self._iostream.close()
self._iostream = None
if err is not None:
raise err
raise socket.error("socket.getaddrinfo returned an empty list")
def test_mset_kwargs(self, r):
d = {'a': b('1'), 'b': b('2'), 'c': b('3')}
assert r.mset(**d)
for k, v in iteritems(d):
assert r[k] == v
def _rc_mset(self, mapping):
"Sets each key in the ``mapping`` dict to its corresponding value"
result = True
for k, v in iteritems(mapping):
result = result and self.set(k, v)
return result
def __init__(self, data):
for k, v in iteritems(data):
self[k.upper()] = v
def from_url(cls, url, db=None, **kwargs):
"""
Return a connection pool configured from the given URL.
For example::
redis://[:password]@localhost:6379/0
unix://[:password]@/path/to/socket.sock?db=0
There are several ways to specify a database number. The parse function
will return the first specified option:
1. A ``db`` querystring option, e.g. redis://localhost?db=0
2. If using the redis:// scheme, the path argument of the url, e.g.
redis://localhost/0
3. The ``db`` argument to this function.
If none of these options are specified, db=0 is used.
Any additional querystring arguments and keyword arguments will be
passed along to the ConnectionPool class's initializer. In the case
of conflicting arguments, querystring arguments always win.
"""
# in python2.6, custom URL schemes don't recognize querystring values
# split the url manually instead
pieces = url.split('?', 1)
url, qs = '', ''
if len(pieces) == 2:
url, qs = pieces
else:
url = pieces[0]
url = urlparse(url)
url_options = {}
for name, value in iteritems(parse_qs(qs)):
if value and len(value) > 0:
url_options[name] = value[0]
# We only support redis:// and unix:// schemes.
if url.scheme == 'unix':
url_options.update({
'password': url.password,
'path': url.path,
'connection_class': UnixDomainSocketConnection,
})
else:
url_options.update({
'host': url.hostname,
'port': int(url.port or 6379),
'password': url.password,
})
# If there's a path argument, use it as the db argument if a
# querystring value wasn't specified
if 'db' not in url_options and url.path:
try:
url_options['db'] = int(url.path.replace('/', ''))
except (AttributeError, ValueError):
pass
# last shot at the db value
url_options['db'] = int(url_options.get('db', db or 0))
# update the arguments from the URL values
kwargs.update(url_options)
return cls(**kwargs)
