def request_to_start_gateway(cluster_name,
server,
leader_cm,
check_state=True):
# add gateway configuration to confmaster
cmd = 'gw_add %s %d %s %s %d' % (cluster_name, server['id'],
server['pm_name'], server['ip'],
server['gateway_port'])
result = util.cm_command(leader_cm['ip'], leader_cm['cm_port'], cmd)
jobj = json.loads(result)
if jobj['state'] != 'success':
util.log('failed to execute. cmd:%s, result:%s' % (cmd, result))
return -1
# start gateway process
ret = util.start_gateway(server['id'], server['ip'], leader_cm['cm_port'],
cluster_name, server['gateway_port'])
if ret is not 0:
util.log('failed to start_gateway. server:%s, id:%d' %
(server['ip'], server['id']))
return -1
# check gateway state
if check_state:
ok = False
try_cnt = 0
while try_cnt < 5:
try_cnt += 1
if util.check_gateway_state(cluster_name, leader_cm, server):
ok = True
break
time.sleep(0.5)
if not ok:
util.log('failed to start_gateway, Invalid state of gateway.' %
inactive_conns)
return -1
# check inactive redis connections of gateway
if check_state:
ok = False
try_cnt = 0
while try_cnt < 10:
try_cnt += 1
if util.gw_info_redis_disccons(server['ip'],
server['gateway_port']) == 0:
ok = True
break
else:
time.sleep(0.5)
if not ok:
util.log(
'failed to start_gateway, invalid number of inactive redis connections. %d'
% inactive_conns)
return -1
util.log('succeeded to start_gateway. server:%s, id:%d' %
(server['ip'], server['id']))
return 0
def test_gateway_lookup( self ):
util.print_frame()
# check initial states
for server in self.cluster['servers']:
success = False
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('cZxid'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# shutdown gateway
for server in self.cluster['servers']:
success = False
ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )
for try_cnt in range( 10 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('Node does not exist'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# restart gateway
for server in self.cluster['servers']:
success = False
ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('cZxid'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
return 0
def test_gateway_lookup( self ):
util.print_frame()
# check initial states
for server in self.cluster['servers']:
success = False
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd(cmd)
if ret['exitcode'] == 'OK':
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# shutdown gateway
for server in self.cluster['servers']:
success = False
ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )
for try_cnt in range( 10 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
if ret['exitcode'] == 'ZK_NO_NODE':
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# restart gateway
for server in self.cluster['servers']:
success = False
ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
if ret['exitcode'] == 'OK':
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
return 0
def request_to_start_gateway( cluster_name, server, leader_cm, check_state=True ):
# add gateway configuration to confmaster
cmd = 'gw_add %s %d %s %s %d' % (cluster_name, server['id'], server['pm_name'], server['ip'], server['gateway_port'])
result = util.cm_command( leader_cm['ip'], leader_cm['cm_port'], cmd )
jobj = json.loads(result)
if jobj['state'] != 'success':
util.log('failed to execute. cmd:%s, result:%s' % (cmd, result))
return -1
# start gateway process
ret = util.start_gateway( server['id'], server['ip'], leader_cm['cm_port'], cluster_name, server['gateway_port'] )
if ret is not 0:
util.log('failed to start_gateway. server:%s, id:%d' % (server['ip'], server['id']))
return -1
# check gateway state
if check_state:
ok = False
try_cnt = 0
while try_cnt < 5:
try_cnt += 1
if util.check_gateway_state( cluster_name, leader_cm, server ):
ok = True
break
time.sleep(0.5)
if not ok:
util.log('failed to start_gateway, Invalid state of gateway.' % inactive_conns)
return -1
# check inactive redis connections of gateway
if check_state:
ok = False
try_cnt = 0
while try_cnt < 10:
try_cnt += 1
if util.gw_info_redis_disccons(server['ip'], server['gateway_port']) == 0:
ok = True
break
else:
time.sleep(0.5)
if not ok:
util.log('failed to start_gateway, invalid number of inactive redis connections. %d' % inactive_conns)
return -1
util.log('succeeded to start_gateway. server:%s, id:%d' % (server['ip'], server['id']))
return 0
def test_gateway_upgrade(self):
util.print_frame()
api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
# Start load generation
self.load_gen_list = {}
for i in range(len(self.cluster['servers'])):
arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
server = self.cluster['servers'][i]
load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10)
load_gen.start()
self.load_gen_list[i] = load_gen
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete gateway
self.assertTrue(
util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
'failed to delete gateway')
# Check load
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Add gateway
self.assertTrue(
util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
'failed to add gateway')
# Check if gateway is added
added_gw = {"ip":HOST_IP,"port":gw_port}
log_reader = LogReader(api.conf.log_file_prefix)
found = False
while True:
line = log_reader.readline()
if line == None:
break
if line.find(MSG_GATEWAY_ADD_ZK) == -1:
continue
gw = line.split('data:')[1]
gw = ast.literal_eval(gw)
if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
found = True
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Check loadbalancing
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
api.destroy()
def _gateway_fault_failback(self, api):
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
time.sleep(3)
# Check error
saved_err_cnt = {}
for i in range(len(self.load_gen_list)):
saved_err_cnt[i] = 0
no_error_cnt = 0
for i in range(10):
util.log('check error count loop:%d' % i)
no_err = True
for j in range(len(self.load_gen_list)):
err_cnt = self.load_gen_list[j].get_err_cnt()
if err_cnt != saved_err_cnt[j]:
no_err = False
util.log('saved_err_cnt:%d, err_cnt:%d' % (saved_err_cnt[j], err_cnt))
saved_err_cnt[j] = err_cnt
if no_err:
no_error_cnt += 1
if no_error_cnt >= 3:
break
time.sleep(1)
self.assertTrue(no_error_cnt >= 3, 'failed to get replys from well working gateways')
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Check if gateway is added
found = check_gateway_added(deleted_gw, api)
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Check loadbalancing
for i in range(3):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
def test_gateway_add_del(self):
util.print_frame()
api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
# Add gateway
gw_port = 10000
gw_id = 10
cmd = 'gw_add %s %d %s %s %d' % (CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port)
ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd)
if ret != None and len(ret) >= 2:
ret = ret[:-2]
util.log('cmd:"%s", ret:"%s"' % (cmd, ret))
if not ret.startswith('{"state":"success","msg":"+OK"}'):
self.fail('failed to add gateway')
# Deploy gateway
server = self.cluster['servers'][0]
ret = util.deploy_gateway(gw_id)
self.assertTrue(ret, 'failed to deploy_gateway')
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(5)
# Check if gateway is added
added_gw = {"ip":HOST_IP,"port":gw_port}
log_reader = LogReader(api.conf.log_file_prefix)
found = False
while True:
line = log_reader.readline()
if line == None:
break
if line.find(MSG_GATEWAY_ADD_ZK) == -1:
continue
gw = line.split('data:')[1]
gw = ast.literal_eval(gw)
if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
found = True
if not found:
self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
else:
util.log('SUCCESS, load gateway information.')
# Delete gateway
cmd = 'gw_del %s %d' % (CLUSTER_NAME, gw_id)
ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd)
if ret != None and len(ret) >= 2:
ret = ret[:-2]
util.log('cmd:"%s", ret:"%s"' % (cmd, ret))
if not ret.startswith('{"state":"success","msg":"+OK"}'):
self.fail('failed to delete gateway')
# Check if gateway is deleted
deleted_gw = {"ip":HOST_IP,"port":gw_port}
found = check_gateway_deleted(deleted_gw, api)
if not found:
self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
else:
util.log('SUCCESS, delete gateway information.')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
api.destroy()
def test_no_error_scenario_and_memory_leak(self):
util.print_frame()
# Start with valgrind
p = util.exec_proc_async('%s/.obj%d' % (constant.ARCCI_DIR, self.arch),
"valgrind ./dummy-perf -z localhost:2181 -c %s -n 5 -s 60" % self.cluster['cluster_name'],
subprocess.PIPE, subprocess.PIPE, subprocess.PIPE);
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete gateway
self.assertTrue(
util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
'failed to delete gateway')
# Check load
for i in range(5):
ok = True
tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Stop gateway
ret = util.shutdown_gateway(gw_id, gw_port, True)
self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
time.sleep(5)
# Start gateway
ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
time.sleep(3)
# Add gateway
self.assertTrue(
util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
'failed to add gateway')
time.sleep(10)
# Check loadbalancing
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
# Check no error
no_memory_leak = False
(stdout, stderr) = p.communicate()
for line in stdout.split("\n"):
print line
if line.find('[ERR]') != -1:
self.fail('find error, msg:%s' % line)
if line.find('All heap blocks were freed -- no leaks are possible'):
no_memory_leak = True
self.assertTrue(no_memory_leak, 'memory leaks are possible')
util.log('no leaks are possible')
