def test_gateway_lookup( self ):
util.print_frame()
# check initial states
for server in self.cluster['servers']:
success = False
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('cZxid'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# shutdown gateway
for server in self.cluster['servers']:
success = False
ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )
for try_cnt in range( 10 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('Node does not exist'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# restart gateway
for server in self.cluster['servers']:
success = False
ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
ret = ret['err']
if -1 != ret.find('cZxid'):
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
return 0
def test_gateway_lookup( self ):
util.print_frame()
# check initial states
for server in self.cluster['servers']:
success = False
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd(cmd)
if ret['exitcode'] == 'OK':
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# shutdown gateway
for server in self.cluster['servers']:
success = False
ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )
for try_cnt in range( 10 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
if ret['exitcode'] == 'ZK_NO_NODE':
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
# restart gateway
for server in self.cluster['servers']:
success = False
ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )
for try_cnt in range( 5 ):
cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
ret = util.zk_cmd( cmd )
if ret['exitcode'] == 'OK':
success = True
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )
return 0
def cleanup_zookeeper_root():
max_try = 20
for i in range(max_try):
print 'try cleanup zookeeper, i:%d' % i
if util.zk_cmd('get /RC')['exitcode'] == 'ZK_NO_NODE':
return 0
ret = util.zk_cmd('rmr /RC')
if ret['exitcode'] == 'OK':
return 0
else:
util.log('failed to clean up zookeeper, ret:%s' % ret)
time.sleep(0.2)
return -1
def cleanup_zookeeper_root():
max_try = 20
for i in range( max_try ):
print 'try cleanup zookeeper, i:%d' % i
if util.zk_cmd( 'get /RC' )['exitcode'] == 'ZK_NO_NODE':
return 0
ret = util.zk_cmd( 'rmr /RC' )
if ret['exitcode'] == 'OK':
return 0
else:
util.log('failed to clean up zookeeper, ret:%s' % ret)
time.sleep(0.2)
return -1
def get_cluster_conf_from_zk(cluster_name):
# Cluster
cluster_conf = {
'clusterName':
cluster_name,
'clusterData':
json.loads(util.zk_cmd('get /RC/CLUSTER/%s' % cluster_name)['data'])
}
# PG
cluster_conf['pgList'] = __znode_to_dict(
'pgId',
util.zk_get_children_with_data('/RC/CLUSTER/%s/PG' %
cluster_conf['clusterName']))
# PGS
cluster_conf['pgsList'] = __znode_to_dict(
'pgsId',
util.zk_get_children_with_data('/RC/CLUSTER/%s/PGS' %
cluster_conf['clusterName']))
# GW
cluster_conf['gwList'] = __znode_to_dict(
'gwId',
util.zk_get_children_with_data('/RC/CLUSTER/%s/GW' %
cluster_conf['clusterName']))
# GW lookup
cluster_conf['gwLookup'] = __znode_to_dict(
'gwId',
util.zk_get_children_with_data('/RC/NOTIFICATION/CLUSTER/%s/GW' %
cluster_conf['clusterName']))
return cluster_conf
def check_gateway_affinity(zone_conf):
for cluster_conf in zone_conf['cluster']:
expected = get_gateway_affinity_cluster(cluster_conf)
affinity = json.loads(util.zk_cmd('get /RC/NOTIFICATION/CLUSTER/%s/AFFINITY' % cluster_conf['clusterName'].encode('ascii'))['data'])
for x, y in zip(expected, affinity):
if len(compare_dict(x, y)) != 0:
util.log('wrong gateway affinity of %s, %s != %s' % (cluster_conf['clusterName'], x, y))
return False
return True
def get_pm_conf_from_zk(pm_name):
return {
'pmName' : pm_name,
'pmInfo' : json.loads(util.zk_cmd('get /RC/PM/%s' % pm_name)['data']),
'clusterList' : sorted(
map(lambda x: {'name' : x['name'] , 'data' : sort_pm_data(json.loads(x['data']))},
util.zk_get_children_with_data('/RC/PM/%s' % pm_name)),
key = lambda x: x['name'])
}
def check_gateway_affinity(zone_conf):
for cluster_conf in zone_conf['cluster']:
expected = get_gateway_affinity_cluster(cluster_conf)
affinity = json.loads(
util.zk_cmd('get /RC/NOTIFICATION/CLUSTER/%s/AFFINITY' %
cluster_conf['clusterName'].encode('ascii'))['data'])
for x, y in zip(expected, affinity):
if len(compare_dict(x, y)) != 0:
util.log('wrong gateway affinity of %s, %s != %s' %
(cluster_conf['clusterName'], x, y))
return False
return True
def get_pm_conf_from_zk(pm_name):
return {
'pmName':
pm_name,
'pmInfo':
json.loads(util.zk_cmd('get /RC/PM/%s' % pm_name)['data']),
'clusterList':
sorted(map(
lambda x: {
'name': x['name'],
'data': sort_pm_data(json.loads(x['data']))
}, util.zk_get_children_with_data('/RC/PM/%s' % pm_name)),
key=lambda x: x['name'])
}
def cleanup_zookeeper_root():
max_try = 20
for i in range( max_try ):
print 'try cleanup zookeeper, i:%d' % i
ret = util.zk_cmd( 'rmr /RC' )
err = ret['err']
if err == '':
return 0
elif -1 != err.find( 'Node does not exist: /RC'):
if err.count('/') == 1:
return 0
else:
util.log('failed to clean up zookeeper, err:%s' % err)
time.sleep(0.2)
return -1
def get_cluster_conf_from_zk(cluster_name):
# Cluster
cluster_conf = {
'clusterName' : cluster_name,
'clusterData' : json.loads(util.zk_cmd('get /RC/CLUSTER/%s' % cluster_name)['data'])
}
# PG
cluster_conf['pgList'] = __znode_to_dict(
'pgId', util.zk_get_children_with_data('/RC/CLUSTER/%s/PG' % cluster_conf['clusterName']))
# PGS
cluster_conf['pgsList'] = __znode_to_dict(
'pgsId', util.zk_get_children_with_data('/RC/CLUSTER/%s/PGS' % cluster_conf['clusterName']))
# GW
cluster_conf['gwList'] = __znode_to_dict(
'gwId', util.zk_get_children_with_data('/RC/CLUSTER/%s/GW' % cluster_conf['clusterName']))
# GW lookup
cluster_conf['gwLookup'] = __znode_to_dict(
'gwId', util.zk_get_children_with_data('/RC/NOTIFICATION/CLUSTER/%s/GW' % cluster_conf['clusterName']))
return cluster_conf
def test_zookeeper_delete_root_of_gw_znodes(self):
util.print_frame()
# Start load generation
self.load_gen_list = {}
arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
server = self.cluster['servers'][0]
load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10, verbose=True)
load_gen.start()
self.load_gen_list[0] = load_gen
# Set up arguments
server = self.cluster['servers'][0]
gw_id = server['id']
gw_port = server['gateway_port']
# Check load
for i in range(5):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
self.assertTrue(ok, 'failed to send requests')
# Delete root of GW znodes
print 'try remove root of GW znodes'
ret = util.zk_cmd('rmr /RC/NOTIFICATION/CLUSTER/%s/GW' % server['cluster_name'])
ret = ret['err']
self.assertEqual(ret, '', 'failed to remove root of GW znodes, ret:%s' % ret)
# Check loadbalancing
for i in range(10):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps > 10:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')
# Recover root of GW znodes
print 'try recover GW znodes'
ret = util.zk_cmd('create /RC/NOTIFICATION/CLUSTER/%s/GW test' % server['cluster_name'])
ret = ret['err']
self.assertNotEqual(ret.find('Created /RC/NOTIFICATION/CLUSTER/testCluster0/GW'), -1,
'failed to create root of GW znodes, ret:%s' % ret)
for s in self.cluster['servers']:
path = '/RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (s['cluster_name'], s['id'])
cmd = 'create %s \'{"ip":"%s","port":%d}\'' % (path, s['ip'], s['gateway_port'])
print cmd
ret = util.zk_cmd(cmd)
ret = ret['err']
self.assertNotEqual(ret.find('Created %s' % path), -1, 'failed to recover GW znode, ret:%s' % ret)
# Check loadbalancing
for i in range(10):
ok = True
for s in self.cluster['servers']:
tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
if tps < 50:
ok = False
if ok:
break
time.sleep(1)
if not ok:
self.fail('FAIL, loadbalancing,')
else:
util.log('SUCCESS, loadbalancing.')