def test_gateway_network_isolation(self):
util.print_frame()
cluster = self.cluster
# Clear rules
while True:
out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
util.log(out)
if out.succeeded == False:
break
while True:
out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
util.log(out)
if out.succeeded == False:
break
while True:
out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
util.log(out)
if out.succeeded == False:
break
# Print rules
out = util.sudo('iptables -L')
util.log('====================================================================')
util.log(out.succeeded)
util.log('out : %s' % out)
util.log('out.return_code : %d' % out.return_code)
util.log('out.stderr : %s' % out.stderr)
# Start loadgenerators
self.load_gen_list = {}
for i in range(len(cluster['servers'])):
arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
server = cluster['servers'][i]
load_gen = LoadGenerator_ARCCI_FaultTolerance(server['id'], arc_api)
load_gen.start()
self.load_gen_list[i] = load_gen
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
# Add virtualhost information to MGMT
VIRTUAL_HOST_NAME = 'virtualhost'
VIRTUAL_HOST_IP = '127.0.0.100'
ret = util.pm_add(VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'pm_add fail.')
# Modify gateway information of MGMT
server = cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Delete gateway
ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_del fail')
# Add gateway
ret= util.gw_add(CLUSTER_NAME, gw_id, VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, gw_port, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_add fail')
# Check load balancing
for i in range(5):
ok = True
for s in cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail')
util.log('load balancing success')
# Block
out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)
# Check blocked gateway`s ops
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - blocked gateway')
util.log('load balancing success - blocked gateway')
# Check unblocked gateway`s ops
for i in range(10):
ok = True
for s in cluster['servers']:
if s == server:
continue
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - nonblocked gateways')
util.log('load balancing success - nonblocked gateways')
# Unblock
out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)
# Check load balancing
ok = False
for i in xrange(5):
condition = (lambda s: (s['ops'] <= 10 if s['id'] == gw_id else s['ops'] >= 50))
if util.check_ops(cluster['servers'], 'gw', condition):
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
util.log('load balancing success - all gateways after unblocking network')
server = cluster['servers'][0]
# Wait until opinion for the gateway deleted.
for i in xrange(5):
util.log('Wait until opinions for the gateway have been deleted... %d' % i)
time.sleep(1)
# Delete gateway
ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_del fail')
# Add gateway
ret= util.gw_add(CLUSTER_NAME, gw_id, server['pm_name'], server['ip'], gw_port, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_add fail')
# Check load balancing
for i in range(10):
ok = True
for s in cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
util.log('load balancing success - all gateways after unblocking network')
def test_gateway_upgrade(self):
util.print_frame()
api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
# Start load generation
self.load_gen_list = {}
for i in range(len(self.cluster['servers'])):
arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
server = self.cluster['servers'][i]
load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10)
load_gen.start()
self.load_gen_list[i] = load_gen
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete gateway
self.assertTrue(
util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
'failed to delete gateway')
# Check load
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Add gateway
self.assertTrue(
util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
'failed to add gateway')
# Check if gateway is added
added_gw = {"ip":HOST_IP,"port":gw_port}
log_reader = LogReader(api.conf.log_file_prefix)
found = False
while True:
line = log_reader.readline()
if line == None:
break
if line.find(MSG_GATEWAY_ADD_ZK) == -1:
continue
gw = line.split('data:')[1]
gw = ast.literal_eval(gw)
if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
found = True
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Check loadbalancing
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
api.destroy()
def test_no_error_scenario_and_memory_leak(self):
util.print_frame()
# Start with valgrind
p = util.exec_proc_async('%s/.obj%d' % (constant.ARCCI_DIR, self.arch),
"valgrind ./dummy-perf -z localhost:2181 -c %s -n 5 -s 60" % self.cluster['cluster_name'],
subprocess.PIPE, subprocess.PIPE, subprocess.PIPE);
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete gateway
self.assertTrue(
util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
'failed to delete gateway')
# Check load
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
time.sleep(5)
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Add gateway
self.assertTrue(
util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
'failed to add gateway')
time.sleep(10)
# Check loadbalancing
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
# Check no error
no_memory_leak = False
(stdout, stderr) = p.communicate()
for line in stdout.split("\n"):
print line
if line.find('[ERR]') != -1:
self.fail('find error, msg:%s' % line)
if line.find('All heap blocks were freed -- no leaks are possible'):
no_memory_leak = True
self.assertTrue(no_memory_leak, 'memory leaks are possible')
util.log('no leaks are possible')
def test_3_heartbeat_target_connection_count(self):
util.print_frame()
util.log('wait until all connections are established')
for i in range(1, 8):
time.sleep(1)
util.log('%d sec' % i)
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(
server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(
server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced' % (self.cluster['cluster_name'],
server['id'])
ret = util.cm_command(self.leader_cm['ip'],
self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual(jobj['state'], 'success',
'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
util.log('succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(
server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to redis%d(%s:%d) is %d, exptected:n<=2, before=%d'
% (server['id'], server['ip'], server['redis_port'], after_cnt,
before_cnt_redis))
util.log(
'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
% (server['id'], server['ip'], server['redis_port'], after_cnt,
before_cnt_redis))
# check smr
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_smr(
server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
% (server['id'], server['ip'], server['smr_mgmt_port'],
after_cnt, expected, before_cnt_smr))
util.log(
'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
% (server['id'], server['ip'], server['smr_mgmt_port'],
after_cnt, expected, before_cnt_smr))
# Go back to initial configuration
self.assertTrue(
util.pgs_join(self.leader_cm['ip'], self.leader_cm['cm_port'],
server['cluster_name'], server['id']),
'failed to join pgs %d' % server['id'])
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'],
server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
ret = util.cm_command(self.leader_cm['ip'],
self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual(jobj['state'], 'success',
'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
util.log('succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(
server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
% (server['id'], server['ip'], server['gateway_port'],
after_cnt, expected))
util.log(
'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
% (server['id'], server['ip'], server['gateway_port'],
after_cnt, expected))
# Go back to initial configuration
self.assertTrue(
util.gw_add(server['cluster_name'], server['id'],
server['pm_name'], server['ip'],
server['gateway_port'], self.leader_cm['ip'],
self.leader_cm['cm_port']),
'failed to add gw %d' % server['id'])
def test_gateway_network_isolation(self):
util.print_frame()
cluster = self.cluster
# Clear rules
while True:
if util.iptables_redirect('D', '127.0.0.100', '127.0.0.1') == False:
break
while True:
if util.iptables_drop('D', '127.0.0.100') == False:
break
# Print rules
util.iptables_print_list()
# Start loadgenerators
self.load_gen_list = {}
for i in range(len(cluster['servers'])):
arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
server = cluster['servers'][i]
load_gen = LoadGenerator_ARCCI_FaultTolerance(server['id'], arc_api)
load_gen.start()
self.load_gen_list[i] = load_gen
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
# Add virtualhost information to MGMT
VIRTUAL_HOST_NAME = 'virtualhost'
VIRTUAL_HOST_IP = '127.0.0.100'
ret = util.pm_add(VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'pm_add fail.')
# Modify gateway information of MGMT
server = cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Delete gateway
ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_del fail')
# Add gateway
ret= util.gw_add(CLUSTER_NAME, gw_id, VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, gw_port, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_add fail')
# Check load balancing
for i in range(5):
ok = True
for s in cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail')
util.log('load balancing success')
# Block
self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')
# Check blocked gateway`s ops
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - blocked gateway')
util.log('load balancing success - blocked gateway')
# Check unblocked gateway`s ops
for i in range(10):
ok = True
for s in cluster['servers']:
if s == server:
continue
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - nonblocked gateways')
util.log('load balancing success - nonblocked gateways')
# Unblock
self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')
# Check load balancing
ok = False
for i in xrange(5):
condition = (lambda s: (s['ops'] <= 10 if s['id'] == gw_id else s['ops'] >= 50))
if util.check_ops(cluster['servers'], 'gw', condition):
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
util.log('load balancing success - all gateways after unblocking network')
server = cluster['servers'][0]
# Wait until opinion for the gateway deleted.
for i in xrange(5):
util.log('Wait until opinions for the gateway have been deleted... %d' % i)
time.sleep(1)
# Delete gateway
ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_del fail')
# Add gateway
ret= util.gw_add(CLUSTER_NAME, gw_id, server['pm_name'], server['ip'], gw_port, MGMT_IP, MGMT_PORT)
self.assertTrue(ret, 'gw_add fail')
# Check load balancing
for i in range(10):
ok = True
for s in cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
util.log('load balancing success - all gateways after unblocking network')
# Go back to initial configuration
self.assertTrue(util.pm_del(MGMT_IP, MGMT_PORT, VIRTUAL_HOST_NAME),
'failed to pm_del to go back to initial configuration.')
def test_3_heartbeat_target_connection_count( self ):
util.print_frame()
util.log( 'wait until all connections are established' )
for i in range(1, 8):
time.sleep(1)
util.log( '%d sec' % i )
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced' % (self.cluster['cluster_name'], server['id'])
ret = util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual( jobj['state'], 'success', 'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
util.log( 'succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected:n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
# check smr
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
# Go back to initial configuration
self.assertTrue(util.pgs_join(self.leader_cm['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['id']),
'failed to join pgs %d' % server['id'])
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
ret = util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual( jobj['state'], 'success', 'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
util.log( 'succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
# Go back to initial configuration
self.assertTrue(util.gw_add(server['cluster_name'], server['id'], server['pm_name'], server['ip'], server['gateway_port'], self.leader_cm['ip'], self.leader_cm['cm_port']),
'failed to add gw %d' % server['id'])
