def test_7_remaining_hbc_connection( self ):
util.print_frame()
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
# check smr
success = False
expected = 0
for i in range(5):
after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
def test_7_remaining_hbc_connection( self ):
util.print_frame()
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
# check smr
success = False
expected = 0
for i in range(5):
after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
def test_3_heartbeat_target_connection_count(self):
util.print_frame()
util.log('wait until all connections are established')
for i in range(1, 8):
time.sleep(1)
util.log('%d sec' % i)
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(
server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(
server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced' % (self.cluster['cluster_name'],
server['id'])
ret = util.cm_command(self.leader_cm['ip'],
self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual(jobj['state'], 'success',
'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
util.log('succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(
server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to redis%d(%s:%d) is %d, exptected:n<=2, before=%d'
% (server['id'], server['ip'], server['redis_port'], after_cnt,
before_cnt_redis))
util.log(
'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
% (server['id'], server['ip'], server['redis_port'], after_cnt,
before_cnt_redis))
# check smr
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_smr(
server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
% (server['id'], server['ip'], server['smr_mgmt_port'],
after_cnt, expected, before_cnt_smr))
util.log(
'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
% (server['id'], server['ip'], server['smr_mgmt_port'],
after_cnt, expected, before_cnt_smr))
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'],
server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
ret = util.cm_command(self.leader_cm['ip'],
self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual(jobj['state'], 'success',
'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
util.log('succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(
server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
% (server['id'], server['ip'], server['gateway_port'],
after_cnt, expected))
util.log(
'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
% (server['id'], server['ip'], server['gateway_port'],
after_cnt, expected))
def test_3_heartbeat_target_connection_count( self ):
util.print_frame()
util.log( 'wait until all connections are established' )
for i in range(1, 8):
time.sleep(1)
util.log( '%d sec' % i )
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced' % (self.cluster['cluster_name'], server['id'])
ret = util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual( jobj['state'], 'success', 'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
util.log( 'succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected:n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
# check smr
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
ret = util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
jobj = json.loads(ret)
self.assertEqual( jobj['state'], 'success', 'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
util.log( 'succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
def test_7_remaining_hbc_connection(self):
util.print_frame()
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(
server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(
server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (
self.cluster['cluster_name'], server['id'],
self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'],
cmd)
for server in self.cluster['servers']:
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(
server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
% (server['id'], server['ip'], server['redis_port'], after_cnt,
before_cnt_redis))
util.log(
'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
% (server['id'], server['ip'], server['redis_port'], after_cnt,
before_cnt_redis))
# check smr
success = False
expected = 0
for i in range(5):
after_cnt = util.get_clients_count_of_smr(
server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
% (server['id'], server['ip'], server['smr_mgmt_port'],
after_cnt, expected, before_cnt_smr))
util.log(
'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
% (server['id'], server['ip'], server['smr_mgmt_port'],
after_cnt, expected, before_cnt_smr))
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'],
server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'],
cmd)
for server in self.cluster['servers']:
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(
server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals(
success, True,
'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
% (server['id'], server['ip'], server['gateway_port'],
after_cnt, expected))
util.log(
'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
% (server['id'], server['ip'], server['gateway_port'],
after_cnt, expected))
# Go back to initial configuration
# Cleanup PG
self.assertTrue(
util.cm_success(
util.cm_command(
self.leader_cm['ip'], self.leader_cm['cm_port'],
'pg_del %s %d' %
(self.cluster['cluster_name'],
self.cluster['servers'][0]['pg_id'])))[0])
# Cleanup processes of PGS and GW
for s in self.cluster['servers']:
self.assertEqual(0, util.shutdown_redis(s['id'], s['redis_port']),
'failed to kill redis %d process' % s['id'])
self.assertEqual(
0, util.shutdown_smr(s['id'], s['ip'], s['smr_base_port']),
'failed to kill smr %d process' % s['id'])
self.assertEqual(0,
util.shutdown_gateway(s['id'], s['gateway_port']),
'failed to kill gw %d process' % s['id'])
# Recover PG
self.assertTrue(
util.install_pg(self.cluster,
self.cluster['servers'],
self.cluster['servers'][0],
start_gw=True),
'failed to recover PGS and GW in a PM')
def test_7_remaining_hbc_connection( self ):
util.print_frame()
# check pgs
for server in self.cluster['servers']:
before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])
cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
# check redis
success = False
for i in range(5):
after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
if after_cnt <= 2:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
# check smr
success = False
expected = 0
for i in range(5):
after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
# check gateway
for server in self.cluster['servers']:
before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
for server in self.cluster['servers']:
success = False
expected = 1
for i in range(5):
after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
if after_cnt == expected:
success = True
break
time.sleep(1)
self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
# Go back to initial configuration
# Cleanup PG
self.assertTrue(util.cm_success(util.cm_command(
self.leader_cm['ip'], self.leader_cm['cm_port'],
'pg_del %s %d' % (self.cluster['cluster_name'], self.cluster['servers'][0]['pg_id'])))[0])
# Cleanup processes of PGS and GW
for s in self.cluster['servers']:
self.assertEqual(0, util.shutdown_redis(s['id'], s['redis_port']),
'failed to kill redis %d process' % s['id'])
self.assertEqual(0, util.shutdown_smr(s['id'], s['ip'], s['smr_base_port']),
'failed to kill smr %d process' % s['id'])
self.assertEqual(0, util.shutdown_gateway(s['id'], s['gateway_port']),
'failed to kill gw %d process' % s['id'])
# Recover PG
self.assertTrue(
util.install_pg(self.cluster, self.cluster['servers'], self.cluster['servers'][0], start_gw=True),
'failed to recover PGS and GW in a PM')