def test_fix(self):
def migrate_one_slot(nodes, _):
if nodes[0].port == 7100:
source, target = nodes
else:
target, source = nodes
return [(source, target, 1)]
comm.start_cluster('127.0.0.1', 7100)
rc = RedisCluster([{'host': '127.0.0.1', 'port': 7100}])
comm.join_cluster('127.0.0.1',
7100,
'127.0.0.1',
7101,
balance_plan=migrate_one_slot)
rc.set('h-893', 'I am in slot 0')
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
t7100 = Talker('127.0.0.1', 7100)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
t7100.talk('cluster', 'setslot', 0, 'importing', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7101 = Talker('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7100.talk('cluster', 'setslot', 0, 'migrating', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
comm.quit_cluster('127.0.0.1', 7101)
rc.delete('h-893')
comm.shutdown_cluster('127.0.0.1', 7100)
t7100.close()
t7101.close()
def set_remotes(proxy_addr, proxy_port, redis_host, redis_port):
time.sleep(1)
t = Talker(proxy_addr, proxy_port)
try:
t.talk('setremotes', redis_host, redis_port)
finally:
t.close()
def _collect_stats(self):
t = Talker(self.details['host'], self.details['port'])
try:
now = time.time()
i = t.talk_raw(CMD_PROXY)
elapse = time.time() - now
lines = i.split('\n')
st = {}
for ln in lines:
k, v = ln.split(':')
st[k] = v
conns = sum([int(c) for c in st['clients_count'].split(',')])
mem_buffer_alloc = sum([int(m) for m in
st['mem_buffer_alloc'].split(',')])
self.details.update({
'stat': True,
'response_time': elapse,
'threads': st['threads'],
'version': st['version'],
'conn': {
'connected_clients': conns,
'completed_commands': int(st['completed_commands']),
'total_process_elapse': float(st['total_process_elapse']),
},
'mem': {'mem_buffer_alloc': mem_buffer_alloc},
})
self.set_available(elapse)
finally:
t.close()
def _collect_stats(self):
t = Talker(self.details['host'], self.details['port'])
try:
node_info = _info_slots(t)
details = _info_detail(t)
node_info['mem'] = {
'used_memory': int(details['used_memory']),
'used_memory_rss': int(details['used_memory_rss']),
'used_memory_human': details['used_memory_human'],
}
node_info['cpu'] = {
'used_cpu_sys': float(details['used_cpu_sys']),
'used_cpu_user': float(details['used_cpu_user']),
'uptime_in_seconds': int(details['uptime_in_seconds']),
}
node_info['conn'] = {
'connected_clients':
int(details['connected_clients']),
'total_commands_processed':
int(details['total_commands_processed']),
}
node_info['storage'] = {
'expired_keys': int(details['expired_keys']),
'evicted_keys': int(details['evicted_keys']),
'keyspace_hits': int(details['keyspace_hits']),
'keyspace_misses': int(details['keyspace_misses']),
'aof_enabled': details['aof_enabled'] == '1',
}
node_info['response_time'] = details['response_time']
node_info['version'] = details['redis_version']
node_info['stat'] = True
self.details.update(node_info)
self.set_available(node_info['response_time'])
finally:
t.close()
def _info_proxy(host, port):
t = Talker(host, port)
try:
now = time.time()
i = t.talk_raw(CMD_PROXY)
elapse = time.time() - now
lines = i.split('\n')
st = {}
for ln in lines:
k, v = ln.split(':')
st[k] = v
conns = sum([int(c) for c in st['clients_count'].split(',')])
mem_buffer_alloc = sum(
[int(m) for m in st['mem_buffer_alloc'].split(',')])
return {
'stat': True,
'response_time': elapse,
'threads': st['threads'],
'version': st['version'],
'conn': {
'connected_clients': conns,
'completed_commands': int(st['completed_commands']),
'total_process_elapse': float(st['total_process_elapse']),
},
'mem': {
'mem_buffer_alloc': mem_buffer_alloc
},
}
finally:
t.close()
def _info_node(host, port):
t = Talker(host, port)
try:
node_info = _info_slots(t)
details = _info_detail(t)
node_info['mem'] = {
'used_memory': int(details['used_memory']),
'used_memory_rss': int(details['used_memory_rss']),
'used_memory_human': details['used_memory_human'],
}
node_info['cpu'] = {
'used_cpu_sys': float(details['used_cpu_sys']),
'used_cpu_user': float(details['used_cpu_user']),
'uptime_in_seconds': int(details['uptime_in_seconds']),
}
node_info['conn'] = {
'connected_clients': int(details['connected_clients']),
'total_commands_processed':
int(details['total_commands_processed']),
}
node_info['storage'] = {
'expired_keys': int(details['expired_keys']),
'evicted_keys': int(details['evicted_keys']),
'keyspace_hits': int(details['keyspace_hits']),
'keyspace_misses': int(details['keyspace_misses']),
'aof_enabled': details['aof_enabled'] == '1',
}
node_info['response_time'] = details['response_time']
node_info['version'] = details['redis_version']
node_info['stat'] = True
return node_info
finally:
t.close()
def recover_by_addr(host, port):
t = Talker(host, port)
try:
m = t.talk_raw(CMD_PING)
if m.lower() != 'pong':
raise ValueError('Expect pong but recv: %s' % m)
node = nm.pick_by(host, port)
finally:
if t is not None:
t.close()
def set_all_nodes_aof(request):
c = models.cluster.get_by_id(request.form['cluster_id'])
if c is None:
raise ValueError('no such cluster')
aof = request.form['aof']
for n in c.nodes:
t = Talker(n.host, n.port)
try:
t.talk('config', 'set', 'appendonly', aof)
finally:
t.close()
def _simple_cmd(host, port, *command):
status = 200
t = Talker(host, port)
try:
r = t.talk(*command)
except ReplyError as e:
r = '-' + e.message
status = 400
finally:
t.close()
return base.json_result(r, status)
def node_exec_command(request):
t = Talker(request.form['host'], int(request.form['port']))
try:
r = t.talk(*json.loads(request.form['cmd']))
except ValueError as e:
r = None if e.message == 'No reply' else ('-ERROR: ' + e.message)
except ReplyError as e:
r = '-' + e.message
finally:
t.close()
return base.json_result(r)
def _collect_stats(self):
t = Talker(self.details['host'], self.details['port'], CONNECT_TIMEOUT)
try:
now = time.time()
i = t.talk_raw(CMD_PROXY)
lines = i.split('\n')
st = {}
for ln in lines:
k, v = ln.split(':', 1)
st[k] = v
conns = sum([int(c) for c in st['clients_count'].split(',')])
mem_buffer_alloc = sum(
[int(m) for m in st['mem_buffer_alloc'].split(',')])
self.details.update({
'stat':
True,
'threads':
st['threads'],
'version':
st['version'],
'used_cpu_sys':
float(st.get('used_cpu_sys', 0)),
'used_cpu_user':
float(st.get('used_cpu_user', 0)),
'conn': {
'connected_clients': conns,
'completed_commands': int(st['completed_commands']),
'total_process_elapse': float(st['total_process_elapse']),
},
'mem': {
'mem_buffer_alloc': mem_buffer_alloc
},
'read_slave':
st.get('read_slave') == '1',
})
if 'last_command_elapse' in st:
self.details['command_elapse'] = max(
[float(x) for x in st['last_command_elapse'].split(',')])
else:
self.details['command_elapse'] = 0
if 'last_remote_cost' in st:
self.details['remote_cost'] = max(
[float(x) for x in st['last_remote_cost'].split(',')])
else:
self.details['remote_cost'] = 0
self.set_available(self.details['command_elapse'])
finally:
t.close()
def proxy_sync_remote(request):
p = models.proxy.get_by_host_port(
request.form['host'], int(request.form['port']))
if p is None or p.cluster is None:
raise ValueError('no such proxy')
cmd = ['setremotes']
for n in p.cluster.nodes:
cmd.extend([n.host, str(n.port)])
t = Talker(p.host, p.port)
try:
t.talk(*cmd)
finally:
t.close()
def cluster_shutdown(request):
c = models.cluster.get_by_id(int(request.form['cluster_id']))
if c is None or len(c.nodes) == 0:
raise ValueError('no such cluster')
n = c.nodes[0]
t = Talker(n.host, n.port)
try:
redistrib.command.shutdown_cluster(n.host, n.port)
n.assignee_id = None
db.session.add(n)
except RedisStatusError as e:
if e.message == 'Cluster containing keys':
raise ValueError('not empty')
raise ValueError(e.message)
except Exception as e:
raise ValueError(e.message)
finally:
t.close()
def node_set_max_mem(request):
max_mem = int(request.form['max_mem'])
if not MAX_MEM_LIMIT[0] <= max_mem <= MAX_MEM_LIMIT[1]:
raise ValueError('invalid max_mem size')
host = request.form['host']
port = int(request.form['port'])
t = None
try:
t = Talker(host, port)
m = t.talk('config', 'set', 'maxmemory', str(max_mem))
if 'ok' != m.lower():
raise ValueError('CONFIG SET maxmemroy redis %s:%d returns %s' %
(host, port, m))
except BaseException as exc:
logging.exception(exc)
raise
finally:
t.close()
def _collect_stats(self, alarm_func):
t = Talker(self.details['host'], self.details['port'], CONNECT_TIMEOUT)
try:
details = _info_detail(t)
cluster_enabled = details.get('cluster_enabled') == '1'
node_info = {'cluster_enabled': cluster_enabled}
if details.get('cluster_enabled') == '1':
node_info.update(_info_slots(t))
node_info.update({
'used_memory': int(details['used_memory']),
'used_memory_rss': int(details['used_memory_rss']),
'used_memory_human': details['used_memory_human'],
})
node_info['maxmemory'] = int(details['maxmemory'])
node_info.update({
'used_cpu_sys': float(details['used_cpu_sys']),
'used_cpu_user': float(details['used_cpu_user']),
'uptime_in_seconds': int(details['uptime_in_seconds']),
})
node_info.update({
'connected_clients': int(details['connected_clients']),
'total_commands_processed': int(
details['total_commands_processed']),
})
node_info.update({
'expired_keys': int(details['expired_keys']),
'evicted_keys': int(details['evicted_keys']),
'keyspace_hits': int(details['keyspace_hits']),
'keyspace_misses': int(details['keyspace_misses']),
'aof_enabled': details['aof_enabled'] == '1',
})
node_info['response_time'] = details['response_time']
node_info['version'] = details['redis_version']
node_info['stat'] = True
self.details.update(node_info)
self.set_available(node_info['response_time'])
finally:
t.close()
try:
self._check_capacity()
except Exception as e:
logging.exception(e)
def setUp(self):
self.t = Talker('127.0.0.1', 27182)
