def request_to_shutdown_gateway( cluster_name, server, leader_cm, check=False ):
ip = server['ip']
port = server['gateway_port']
id = server['id']
# delete gateway configuration from confmaster
cmd = 'gw_del %s %d' % (cluster_name, id)
result = util.cm_command( leader_cm['ip'], leader_cm['cm_port'], cmd )
jobj = json.loads(result)
if jobj['state'] != 'success':
util.log('failed to execute. cmd:%s, result:%s' % (cmd, result))
return -1
# check client connection
ok = False
for i in range(10):
client_conn_num = util.gw_info_client_cnt(ip, port)
if client_conn_num == 1:
ok = True
break
else:
time.sleep(1)
if ok == False:
util.log('failed to shutdown_gateway, invalid number of client connections. %d' % (client_conn_num - 1))
return -1
# shutdown gateway process
if util.shutdown_gateway( id, port ) is not 0:
util.log('failed to shutdown_gateway %d' % (id))
return -1
util.log('succeeded to shutdown_gateway. %d' % (id))
return 0
def test_gateway_lookup( self ):
util.print_frame()
# check initial states
for server in self.cluster['servers']:
success = False
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('cZxid'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# shutdown gateway
for server in self.cluster['servers']:
success = False
ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )
for try_cnt in range( 10 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('Node does not exist'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# restart gateway
for server in self.cluster['servers']:
success = False
ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('cZxid'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
return 0
def test_gateway_upgrade(self):
util.print_frame()
api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
# Start load generation
self.load_gen_list = {}
for i in range(len(self.cluster['servers'])):
arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
server = self.cluster['servers'][i]
load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10)
load_gen.start()
self.load_gen_list[i] = load_gen
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete gateway
self.assertTrue(
util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
'failed to delete gateway')
# Check load
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Add gateway
self.assertTrue(
util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
'failed to add gateway')
# Check if gateway is added
added_gw = {"ip":HOST_IP,"port":gw_port}
log_reader = LogReader(api.conf.log_file_prefix)
found = False
while True:
line = log_reader.readline()
if line == None:
break
if line.find(MSG_GATEWAY_ADD_ZK) == -1:
continue
gw = line.split('data:')[1]
gw = ast.literal_eval(gw)
if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
found = True
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Check loadbalancing
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
api.destroy()
def _gateway_fault_failback(self, api):
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
time.sleep(3)
# Check error
saved_err_cnt = {}
for i in range(len(self.load_gen_list)):
saved_err_cnt[i] = 0
no_error_cnt = 0
for i in range(10):
util.log('check error count loop:%d' % i)
no_err = True
for j in range(len(self.load_gen_list)):
err_cnt = self.load_gen_list[j].get_err_cnt()
if err_cnt != saved_err_cnt[j]:
no_err = False
util.log('saved_err_cnt:%d, err_cnt:%d' % (saved_err_cnt[j], err_cnt))
saved_err_cnt[j] = err_cnt
if no_err:
no_error_cnt += 1
if no_error_cnt >= 3:
break
time.sleep(1)
self.assertTrue(no_error_cnt >= 3, 'failed to get replys from well working gateways')
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Check if gateway is added
found = check_gateway_added(deleted_gw, api)
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Check loadbalancing
for i in range(3):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
def test_gateway_add_del(self):
util.print_frame()
api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
# Add gateway
gw_port = 10000
gw_id = 10
cmd = 'gw_add %s %d %s %s %d' % (CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port)
ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd)
if ret != None and len(ret) >= 2:
ret = ret[:-2]
util.log('cmd:"%s", ret:"%s"' % (cmd, ret))
if not ret.startswith('{"state":"success","msg":"+OK"}'):
self.fail('failed to add gateway')
# Deploy gateway
server = self.cluster['servers'][0]
ret = util.deploy_gateway(gw_id)
self.assertTrue(ret, 'failed to deploy_gateway')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(5)
# Check if gateway is added
added_gw = {"ip":HOST_IP,"port":gw_port}
log_reader = LogReader(api.conf.log_file_prefix)
found = False
while True:
line = log_reader.readline()
if line == None:
break
if line.find(MSG_GATEWAY_ADD_ZK) == -1:
continue
gw = line.split('data:')[1]
gw = ast.literal_eval(gw)
if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
found = True
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Delete gateway
cmd = 'gw_del %s %d' % (CLUSTER_NAME, gw_id)
ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd)
if ret != None and len(ret) >= 2:
ret = ret[:-2]
util.log('cmd:"%s", ret:"%s"' % (cmd, ret))
if not ret.startswith('{"state":"success","msg":"+OK"}'):
self.fail('failed to delete gateway')
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
api.destroy()
def test_no_error_scenario_and_memory_leak(self):
util.print_frame()
# Start with valgrind
p = util.exec_proc_async('%s/.obj%d' % (constant.ARCCI_DIR, self.arch),
"valgrind ./dummy-perf -z localhost:2181 -c %s -n 5 -s 60" % self.cluster['cluster_name'],
subprocess.PIPE, subprocess.PIPE, subprocess.PIPE);
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete gateway
self.assertTrue(
util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
'failed to delete gateway')
# Check load
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
time.sleep(5)
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Add gateway
self.assertTrue(
util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
'failed to add gateway')
time.sleep(10)
# Check loadbalancing
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
# Check no error
no_memory_leak = False
(stdout, stderr) = p.communicate()
for line in stdout.split("\n"):
print line
if line.find('[ERR]') != -1:
self.fail('find error, msg:%s' % line)
if line.find('All heap blocks were freed -- no leaks are possible'):
no_memory_leak = True
self.assertTrue(no_memory_leak, 'memory leaks are possible')
util.log('no leaks are possible')
def test_7_remaining_hbc_connection( self ):
util.print_frame()
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
# check smr
success = False
expected = 0
for i in range(5):
after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
# Go back to initial configuration
# Cleanup PG
self.assertTrue(util.cm_success(util.cm_command(
self.leader_cm['ip'], self.leader_cm['cm_port'],
'pg_del %s %d' % (self.cluster['cluster_name'], self.cluster['servers'][0]['pg_id'])))[0])
# Cleanup processes of PGS and GW
for s in self.cluster['servers']:
self.assertEqual(0, util.shutdown_redis(s['id'], s['redis_port']),
'failed to kill redis %d process' % s['id'])
self.assertEqual(0, util.shutdown_smr(s['id'], s['ip'], s['smr_base_port']),
'failed to kill smr %d process' % s['id'])
self.assertEqual(0, util.shutdown_gateway(s['id'], s['gateway_port']),
'failed to kill gw %d process' % s['id'])
# Recover PG
self.assertTrue(
util.install_pg(self.cluster, self.cluster['servers'], self.cluster['servers'][0], start_gw=True),
'failed to recover PGS and GW in a PM')