def set_all_nodes_aof():
c = models.cluster.get_by_id(request.form['cluster_id'])
if c is None:
raise ValueError('no such cluster')
aof = request.form['aof']
for n in c.nodes:
with Connection(n.host, n.port) as t:
t.talk('config', 'set', 'appendonly', aof)
return ''
def proxy_sync_remote():
p = models.proxy.get_by_host_port(
request.form['host'], int(request.form['port']))
if p is None or p.cluster is None:
raise ValueError('no such proxy')
cmd = ['setremotes']
for n in p.cluster.nodes:
cmd.extend([n.host, str(n.port)])
with Connection(p.host, p.port) as t:
t.talk(*cmd)
return ''
def set_aof():
aof = 'yes' if request.form['aof'] == 'y' else 'no'
host = request.form['host']
port = int(request.form['port'])
models.audit.raw_event(host, port, models.audit.EVENT_TYPE_CONFIG,
bp.app.get_user_id(), {'aof': aof})
with Connection(host, port) as t:
m = t.talk('config', 'set', 'appendonly', aof)
if 'ok' != m.lower():
raise ValueError('CONFIG SET APPENDONLY redis %s:%d returns %s' %
(host, port, m))
def _simple_cmd(host, port, *command):
status = 200
try:
with Connection(host, port) as t:
try:
r = t.talk(*command)
except ReplyError as e:
r = {'reason': e.message}
status = 400
except IOError:
status = 400
r = {'reason': 'not reachable'}
return json_response(r, status)
def set_max_mem():
max_mem = int(request.form['max_mem'])
if not MAXMEM_LIMIT_LOW <= max_mem <= bp.app.config_node_max_mem:
raise ValueError('invalid max_mem size')
host = request.form['host']
port = int(request.form['port'])
models.audit.raw_event(host, port, models.audit.EVENT_TYPE_CONFIG,
bp.app.get_user_id(), {'max_mem': max_mem})
with Connection(host, port) as t:
m = t.talk('config', 'set', 'maxmemory', str(max_mem))
if 'ok' != m.lower():
raise ValueError('CONFIG SET MAXMEMROY redis %s:%d returns %s' %
(host, port, m))
def cluster_shutdown():
c = models.cluster.get_by_id(int(request.form['cluster_id']))
if c is None or len(c.nodes) == 0:
raise ValueError('no such cluster')
n = c.nodes[0]
with Connection(n.host, n.port) as t:
try:
redistrib.command.shutdown_cluster(n.host, n.port)
n.assignee_id = None
db.session.add(n)
except RedisStatusError as e:
if e.message == 'Cluster containing keys':
raise ValueError('not empty')
raise ValueError(e.message)
except Exception as e:
raise ValueError(e.message)
def _collect_stats(self):
with Connection(self.host, self.port, CONNECT_TIMEOUT) as t:
i = t.talk_raw(CMD_PROXY)
lines = i.split('\n')
st = {}
for ln in lines:
k, v = ln.split(':', 1)
st[k] = v
conns = sum([int(c) for c in st['clients_count'].split(',')])
mem_buffer_alloc = sum([int(m) for m in
st['mem_buffer_alloc'].split(',')])
cluster_ok = st.get('cluster_ok') != '0'
self.details.update({
'stat': cluster_ok,
'threads': st['threads'],
'version': st['version'],
'used_cpu_sys': float(st.get('used_cpu_sys', 0)),
'used_cpu_user': float(st.get('used_cpu_user', 0)),
'connected_clients': conns,
'completed_commands': int(st['completed_commands']),
'total_process_elapse': float(st['total_process_elapse']),
'mem_buffer_alloc': mem_buffer_alloc,
'read_slave': st.get('read_slave') == '1',
'cluster_ok': cluster_ok,
})
if 'last_command_elapse' in st:
self.details['command_elapse'] = max(
[float(x) for x in st['last_command_elapse'].split(',')])
else:
self.details['command_elapse'] = 0
if 'last_remote_cost' in st:
self.details['remote_cost'] = max(
[float(x) for x in st['last_remote_cost'].split(',')])
else:
self.details['remote_cost'] = 0
if cluster_ok:
self.set_available()
else:
self.send_alarm(
'Cluster failed behind %s:%d' % (self.host, self.port),
None)
self.set_unavailable()
def exec_command():
host = request.form['host']
port = int(request.form['port'])
args = json.loads(request.form['cmd'])
models.audit.raw_event(host, port, models.audit.EVENT_TYPE_EXEC,
bp.app.get_user_id(), args)
try:
with Connection(host, port) as t:
try:
r = t.talk(*args)
except ValueError as e:
r = None if e.message == 'No reply' else ('-ERROR: ' +
e.message)
except ReplyError as e:
r = '-' + e.message
except Exception as e:
r = '!ERROR: ' + (e.message or ('%s' % e))
return r
def add_redis():
host = request.form['host']
port = int(request.form['port'])
# do some necessary validation
status = 0
reason = 'success'
try:
with Connection(host, port) as t:
try:
info = t.talk("info")
info_dict = parse_config(info)
if info_dict['cluster_enabled'] == '0':
status = 3
reason = 'not in cluster mode'
except ReplyError as e:
reason = e.message
status = 2
except IOError, e:
status = 1
reason = e.message
def _collect_stats(self):
with Connection(self.details['host'], self.details['port'],
CONNECT_TIMEOUT) as t:
details = _info_detail(t)
cluster_enabled = details.get('cluster_enabled') == '1'
node_info = {'cluster_enabled': cluster_enabled}
if details.get('cluster_enabled') == '1':
node_info.update(_info_slots(t))
node_info.update({
'used_memory': int(details['used_memory']),
'used_memory_rss': int(details['used_memory_rss']),
'used_memory_human': details['used_memory_human'],
})
node_info['maxmemory'] = int(details['maxmemory'])
node_info.update({
'used_cpu_sys': float(details['used_cpu_sys']),
'used_cpu_user': float(details['used_cpu_user']),
'uptime_in_seconds': int(details['uptime_in_seconds']),
})
node_info.update({
'connected_clients': int(details['connected_clients']),
'total_commands_processed': int(
details['total_commands_processed']),
})
node_info.update({
'expired_keys': int(details['expired_keys']),
'evicted_keys': int(details['evicted_keys']),
'keyspace_hits': int(details['keyspace_hits']),
'keyspace_misses': int(details['keyspace_misses']),
'aof_enabled': details['aof_enabled'] == '1',
})
node_info['keys'] = self._keys(details.get('db0', ''))
node_info['version'] = details['redis_version']
node_info['stat'] = True
self.details.update(node_info)
self.set_available()
try:
self._check_capacity()
except Exception as e:
logging.exception(e)
def proxy_sync_remote():
p = models.proxy.get_by_host_port(
request.form['host'], int(request.form['port']))
if p is None or p.cluster is None:
raise ValueError('no such proxy')
if p.proxy_type == TYPE_CORVUS:
cmd = ['config', 'set', 'node']
hosts = ''
for n in p.cluster.nodes:
if len(hosts) > 0:
hosts += ','
hosts += n.host + ':' + str(n.port)
cmd.extend([hosts])
print cmd
else:
cmd = ['setremotes']
for n in p.cluster.nodes:
cmd.extend([n.host, str(n.port)])
with Connection(p.host, p.port) as t:
t.talk(*cmd)
return ''
def test_fix(self):
def migrate_one_slot(nodes, _):
if nodes[0].port == 7100:
source, target = nodes
else:
target, source = nodes
return [(source, target, 1)]
comm.create([('127.0.0.1', 7100)])
rc = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
comm.join_cluster('127.0.0.1',
7100,
'127.0.0.1',
7101,
balance_plan=migrate_one_slot)
rc.set('h-893', 'I am in slot 0')
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
t7100 = Connection('127.0.0.1', 7100)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
t7100.execute('cluster', 'setslot', 0, 'importing', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7101 = Connection('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7100.execute('cluster', 'setslot', 0, 'migrating', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
comm.quit_cluster('127.0.0.1', 7101)
rc.delete('h-893')
comm.shutdown_cluster('127.0.0.1', 7100)
t7100.close()
t7101.close()
def test_fix(self):
def migrate_one_slot(nodes, _):
if nodes[0].port == 7100:
source, target = nodes
else:
target, source = nodes
return [(source, target, 1)]
comm.create([('127.0.0.1', 7100)])
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
comm.join_cluster(
'127.0.0.1',
7100,
'127.0.0.1',
7101,
balance_plan=migrate_one_slot)
rc.set('h-893', 'I am in slot 0')
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
t7100 = Connection('127.0.0.1', 7100)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
t7100.execute('cluster', 'setslot', 0, 'importing', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7101 = Connection('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7100.execute('cluster', 'setslot', 0, 'migrating', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
comm.quit_cluster('127.0.0.1', 7101)
rc.delete('h-893')
comm.shutdown_cluster('127.0.0.1', 7100)
t7100.close()
t7101.close()
def set_remotes():
time.sleep(1)
with Connection(proxy_addr, proxy_port) as t:
t.talk('SETREMOTES', redis_host, redis_port)
