def test_refresh_table_asap():
"""
If this variable is set externally, initialize() should be called.
"""
with patch.object(NodeManager, "initialize") as mock_initialize:
mock_initialize.return_value = None
# Patch parse_response to avoid issues when the cluster sometimes return MOVED
with patch.object(StrictRedisCluster, "parse_response") as mock_parse_response:
def side_effect(self, *args, **kwargs):
return None
mock_parse_response.side_effect = side_effect
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = [
{"host": "127.0.0.1", "port": 7002, "name": "127.0.0.1:7002", "server_type": "master"}
]
r.refresh_table_asap = True
i = len(mock_initialize.mock_calls)
r.execute_command("SET", "foo", "bar")
assert len(mock_initialize.mock_calls) - i == 1
assert r.refresh_table_asap is False
def monkey_link(host=None, port=None, decode_responses=False):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004]]]
elif port == 7001:
result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]],
[5461, 10922, [b'127.0.0.1', 7000], [b'127.0.0.1', 7004]]]
else:
result = []
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
def execute_command(*args, **kwargs):
if args == ("cluster", "slots"):
return result
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
m = Mock(autospec=True)
def ask_redirect_effect(connection, command_name, **options):
def ok_response(connection, command_name, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7001
return "MOCK_OK"
m.side_effect = ok_response
raise AskError("1337 127.0.0.1:7001")
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes['127.0.0.1:7001'] = {
'host': u'127.0.0.1',
'server_type': 'master',
'port': 7001,
'name': '127.0.0.1:7001'
}
with patch.object(StrictRedisCluster, 'parse_response') as parse_response:
host_ip = find_node_ip_based_on_port(r, '7001')
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
parse_response.side_effect = ask_redirect_effect
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def monkey_link(host=None, port=None, *args, **kwargs):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7001],
[b'127.0.0.1', 7004]
]]
elif port == 7001:
result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7000],
[b'127.0.0.1', 7004]
]]
else:
result = []
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
def execute_command(*args, **kwargs):
if args == ("cluster", "slots"):
return result
elif args == ('CONFIG GET', 'cluster-require-full-coverage'):
return {'cluster-require-full-coverage': 'yes'}
else:
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes["127.0.0.1:7001"] = {
"host": u"127.0.0.1",
"server_type": "master",
"port": 7001,
"name": "127.0.0.1:7001",
}
with patch.object(StrictRedisCluster, "parse_response") as parse_response:
host_ip = find_node_ip_based_on_port(r, "7001")
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
parse_response.side_effect = ask_redirect_effect
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes['127.0.0.1:7001'] = {
'host': u'127.0.0.1',
'server_type': 'master',
'port': 7001,
'name': '127.0.0.1:7001'
}
m = Mock(autospec=True)
host_ip = find_node_ip_based_on_port(r, '7001')
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
m.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
m = Mock(autospec=True)
def ask_redirect_effect(connection, command_name, **options):
def ok_response(connection, command_name, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7001
return "MOCK_OK"
m.side_effect = ok_response
resp = ResponseError()
resp.message = "ASK 1337 127.0.0.1:7001"
raise resp
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_refresh_table_asap():
"""
If this variable is set externally, initialize() should be called.
"""
with patch.object(NodeManager, 'initialize') as mock_initialize:
mock_initialize.return_value = None
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = {
"host": "127.0.0.1",
"port": 7002,
"name": "127.0.0.1:7002",
"server_type": "master",
}
r.refresh_table_asap = True
i = len(mock_initialize.mock_calls)
r.execute_command("SET", "foo", "bar")
assert len(mock_initialize.mock_calls) - i == 1
assert r.refresh_table_asap is False
def test_refresh_table_asap():
"""
If this variable is set externally, initialize() should be called.
"""
with patch.object(NodeManager, 'initialize') as mock_initialize:
mock_initialize.return_value = None
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = {
"host": "127.0.0.1",
"port": 7002,
"name": "127.0.0.1:7002",
"server_type": "master",
}
r.refresh_table_asap = True
i = len(mock_initialize.mock_calls)
r.execute_command("SET", "foo", "bar")
assert len(mock_initialize.mock_calls) - i == 1
assert r.refresh_table_asap is False
def main():
global VERBOSE
VERBOSE = "verbose" in sys.argv[1:]
subprocess.check_call(["cargo", "build"])
cluster_sz = 3
cluster = [Instance(i, cluster_sz) for i in range(cluster_sz)]
cluster[0].cluster_init()
cluster[1].cluster_join()
cluster[2].cluster_join()
cluster[0].execute("CLUSTER", "REBALANCE")
time.sleep(5)
client = StrictRedisCluster(
startup_nodes=[{
"host": n.listen_addr.partition(":")[0],
"port": int(n.listen_addr.partition(":")[2])
} for n in cluster],
decode_responses=False,
socket_timeout=0.5,
)
check_map = defaultdict(set)
items = 1000
groups = 100
for i in xrange(items):
k = str(i % groups)
v = str(i)
client.execute_command("SET", k, v, "", "Q")
check_map[k].add(v)
if random.random() < 0.1:
n = random.choice(cluster)
n.restart()
n.wait_ready(lambda c: c.execute_command("CLUSTER", "CONNECTIONS"))
time.sleep(5)
for k, expected in check_map.items():
values = set(client.get(k)[:-1])
assert values == expected, "%s %s %s" % (k, expected, values)
for c in cluster:
values = set(c.client.execute_command("GET", k, "1")[:-1])
assert values == expected, "%s %s %s" % (k, expected, values)
# {"host": "192.168.0.178", "port": "7001"},\
{"host": "192.168.0.179", "port": "7000"}, \
{"host": "192.168.0.179", "port": "7001"},\
{"host": "192.168.0.180", "port": "7000"},\
{"host": "192.168.0.180", "port": "7001"}]
# {u'SliceSize': 166, u'DurationCP': 0.000751436, u'DurationCMP': 3.7068e-05, u'ServerIp': u'192.168.0.171', u'DurationML': 0.000553802, u'DurationNTT': 3.7041e-05, u'DurationRS': 0.001379347}
layerrecipe = 'Layer:Recipe::sha256:be34613ab80dd380c9b007be3e57ac47bca5c1840570ad30b7245ce44199628c'
recipe = 'Blob:File:Recipe::sha256*'
f = '*be34613ab80dd380c9b007be3e57ac47bca5c1840570ad30b7245ce44199628c*'
ulmap = 'ULMap::*'
rlmap = 'RLMap::*'
rj_dbNoBFRecipe = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
bfrecipe = rj_dbNoBFRecipe.execute_command('GET', layerrecipe)
print bfrecipe
#for key in rj_dbNoBFRecipe.scan_iter(layerrecipe):
# k = key.split('::')[1]
# #print k
# bfrecipe = json.loads(rj_dbNoBFRecipe.execute_command('GET', key))
# print(k, bfrecipe)
# try:
# dt = bfrecipe['SliceSizeMap']
# #print dt
# s = 0
# for k, v in dt.items():
# s += v
# if v >= 100*1024*1024:
