def __getitem__(self, index):
with redis.StrictRedis(connection_pool=CXN) as rdb:
if not rdb.exists(self.name):
raise redis.DataError(f'Dataset named {self.name} does not exist')
if index >= rdb.llen(self.name):
raise IndexError
with io.BytesIO(rdb.lindex(self.name, index)) as buf:
return torch.load(buf)
def hmset(self, name, mapping):
"""
Sets each key in the ``mapping`` dict to its corresponding value
in the hash ``name``
"""
if not mapping:
raise redis.DataError("'hmset' with 'mapping' of length 0")
self._db.setdefault(name, {}).update(mapping)
return True
def hmset(self, name, mapping):
"""
Sets each key in the ``mapping`` dict to its corresponding value
in the hash ``name``
"""
if not mapping:
raise redis.DataError("'hmset' with 'mapping' of length 0")
for k, v in mapping.items():
mapping[k] = to_bytes(v)
self._db.setdefault(name, _StrKeyDict()).update(mapping)
return True
def get(self, key):
with redis.StrictRedis(connection_pool=CXN) as rdb:
if rdb.exists(key):
return RedisListObject(key)
else:
raise redis.DataError(f'Dataset named {key} does not exist')
def delete(self):
with redis.StrictRedis(connection_pool=CXN) as rdb:
if rdb.exists(self.name):
rdb.delete(self.name)
else:
raise redis.DataError(f'Dataset named {self.name} does not exist')
def function(*args, **kwargs):
if name not in StrictRedisCluster._loop_keys:
# take care of hash tags
tag_start = None
key_type = hash_tag = ''
# since we don't have "first item" in dict,
# this list is needed in order to check hash_tag in mset({"a{a}": "a", "b":"b"})
list_ht = []
if isinstance(args[0], (basestring, bytes)):
key_type = 'string'
list_ht.append(args[0])
else:
if isinstance(args[0], list):
key_type = 'list'
list_ht.append(args[0][0])
else:
key_type = 'dict'
list_ht = dictkeys(args[0])
# check for hash tags
for k in list_ht:
try:
tag_start = k.index('{')
hash_tag = k
break
except Exception as e:
tag_start = None
# trigger error msg on tag keys unless we have hash tags e.g. "bar{zap}"
if name in StrictRedisCluster._tag_keys and not tag_start:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError(
"rediscluster: Command %s Not Supported (each key name has its own node)"
% name)
# get the hash key
hkey = args[0]
# take care of hash tags names for forcing multiple keys on the same node,
# e.g. r.set("bar{zap}", "bar"), r.mget(["foo{foo}","bar"])
if tag_start is not None:
L = list(args)
if key_type != 'string':
if key_type == 'list':
hkey = L[0][0][tag_start + 1:-1]
L[0][0] = L[0][0][0:tag_start]
else:
hkey = hash_tag[tag_start + 1:-1]
L[0][hash_tag[0:tag_start]] = L[0][hash_tag]
del L[0][hash_tag]
else:
hkey = L[0][tag_start + 1:-1]
L[0] = L[0][0:tag_start]
args = tuple(L)
# get the node number
node = self._getnodenamefor(hkey)
if name in StrictRedisCluster._write_keys:
redisent = self.redises[node]
elif name in StrictRedisCluster._read_keys:
redisent = self.redises[node + '_slave']
else:
raise redis.DataError(
"rediscluster: Command %s Not Supported (each key name has its own node)"
% name)
# Execute the command on the server
return getattr(redisent, name)(*args, **kwargs)
else:
# take care of keys that don't need to go through master and slaves redis servers
if name not in self._loop_keys_admin:
try:
return getattr(self, '_rc_' + name)(*args, **kwargs)
except AttributeError:
raise redis.DataError(
"rediscluster: Command %s Not Supported (each key name has its own node)"
% name)
result = {}
for alias, redisent in iteritems(self.redises):
if (name in StrictRedisCluster._write_keys
and alias.find('_slave') >= 0) or (
name in StrictRedisCluster._read_keys
and alias.find('_slave') == -1):
res = None
else:
res = getattr(redisent, name)(*args, **kwargs)
result[alias] = res
return result
def __init__(self, cluster={}, db=0, mastersonly=False):
# raise exception when wrong server hash
if 'nodes' not in cluster:
raise Exception(
"rediscluster: Please set a correct array of redis cluster.")
self.cluster = cluster
have_master_of = 'master_of' in self.cluster
self.no_servers = len(
self.cluster['master_of']) if have_master_of else len(
self.cluster['nodes'])
self.redises = {}
redises_cons = {}
self.cluster['slaves'] = {}
# connect to all servers
for alias, server in iteritems(self.cluster['nodes']):
if have_master_of and alias not in self.cluster['master_of']:
continue
server_str = str(server)
if server_str in redises_cons:
self.redises[alias] = redises_cons[server_str]['master']
self.redises[alias +
'_slave'] = redises_cons[server_str]['slave']
self.cluster['slaves'][
alias + '_slave'] = redises_cons[server_str]['slave_node']
else:
try:
# connect to master
self.__redis = redis.StrictRedis(db=db, **server)
if not mastersonly and not have_master_of:
info = self.__redis.info()
if info['role'] != 'master':
raise redis.DataError(
"rediscluster: server %s is not a master." %
(server, ))
self.redises[alias] = self.__redis
redises_cons[server_str] = {}
redises_cons[server_str]['master'] = self.redises[alias]
# connect to slave
slave_connected = False
slave = {}
if not mastersonly:
if have_master_of:
slave = self.cluster['nodes'][
self.cluster['master_of'][alias]]
elif 'connected_slaves' in info and info[
'connected_slaves'] > 0:
slave_host, slave_port, slave_online = info[
'slave0'].split(',')
if slave_online == 'online':
slave = {
'host': slave_host,
'port': slave_port
}
if slave:
try:
redis_slave = redis.StrictRedis(host=slave['host'],
port=int(
slave['port']),
db=db)
self.redises[alias + '_slave'] = redis_slave
self.cluster['slaves'][alias + '_slave'] = {
'host': slave['host'],
'port': slave['port']
}
redises_cons[server_str]['slave'] = self.redises[
alias + '_slave']
redises_cons[server_str][
'slave_node'] = self.cluster['slaves'][
alias + '_slave']
slave_connected = True
except redis.RedisError as e:
pass
# "RedisCluster cannot connect to: " + slave_host +':'+ slave_port
if not slave_connected:
self.redises[alias + '_slave'] = self.redises[alias]
self.cluster['slaves'][alias + '_slave'] = server
redises_cons[server_str]['slave'] = self.redises[
alias + '_slave']
redises_cons[server_str]['slave_node'] = self.cluster[
'slaves'][alias + '_slave']
except redis.RedisError as e:
raise redis.ConnectionError(
"rediscluster cannot connect to: %s %s" % (server, e))
