def test_quorum_policy_of_hanging_master( self ):
util.print_frame()
# get master, slave1, slave2
m, s1, s2 = util.get_mss( self.cluster )
self.assertNotEqual( m, None, 'master is None.' )
self.assertNotEqual( s1, None, 'slave1 is None.' )
self.assertNotEqual( s2, None, 'slave2 is None.' )
# hang
smr = smr_mgmt.SMR( m['id'] )
ret = smr.connect( m['ip'], m['smr_mgmt_port'] )
self.assertEqual( ret, 0, 'failed to connect to master. %s:%d' % (m['ip'], m['smr_mgmt_port']) )
smr.write( 'fi delay sleep 1 15000\r\n' )
time.sleep( 5 )
# wait for forced master election
success = False
new_master = None
for i in range( 7 ):
role = util.get_role_of_server( s1 )
if role == c.ROLE_MASTER:
success = True
new_master = s1
break
role = util.get_role_of_server( s2 )
if role == c.ROLE_MASTER:
success = True
new_master = s2
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed to forced master election' )
# shutdown confmaster
for server in self.cluster['servers']:
util.shutdown_cm( server['id'] )
# wait until hanging master wake up
time.sleep( 5 )
# check quorum policy
quorum_of_haning_master = util.get_quorum( m )
self.assertEqual( 2, quorum_of_haning_master,
'invalid quorum of haning master, expected:%d, but:%d' %(2, quorum_of_haning_master) )
util.log( 'succeeded : quorum of haning master=%d' % quorum_of_haning_master )
# check quorum policy
quorum_of_new_master = util.get_quorum( new_master )
self.assertNotEqual( None, quorum_of_new_master, 'failed : find new master' )
self.assertEqual( 1, quorum_of_new_master ,
'invalid quorum of new master, expected:%d, but:%d' % (1, quorum_of_new_master) )
util.log( 'succeeded : quorum of new master=%d' % quorum_of_new_master )
# Go back to initial configuration
# Recover Confmaster
self.assertTrue(util.recover_confmaster(self.cluster, [0,1,2], 0), 'failed to recover confmaster')
return 0
def test_quorum_policy_of_hanging_master( self ):
util.print_frame()
# get master, slave1, slave2
m, s1, s2 = util.get_mss( self.cluster )
self.assertNotEqual( m, None, 'master is None.' )
self.assertNotEqual( s1, None, 'slave1 is None.' )
self.assertNotEqual( s2, None, 'slave2 is None.' )
# hang
smr = smr_mgmt.SMR( m['id'] )
ret = smr.connect( m['ip'], m['smr_mgmt_port'] )
self.assertEqual( ret, 0, 'failed to connect to master. %s:%d' % (m['ip'], m['smr_mgmt_port']) )
smr.write( 'fi delay sleep 1 15000\r\n' )
time.sleep( 5 )
# wait for forced master election
success = False
new_master = None
for i in range( 7 ):
role = util.get_role_of_server( s1 )
if role == c.ROLE_MASTER:
success = True
new_master = s1
break
role = util.get_role_of_server( s2 )
if role == c.ROLE_MASTER:
success = True
new_master = s2
break
time.sleep( 1 )
self.assertEqual( success, True, 'failed to forced master election' )
# shutdown confmaster
for server in self.cluster['servers']:
util.shutdown_cm( server['id'] )
# wait until hanging master wake up
time.sleep( 5 )
# check quorum policy
quorum_of_haning_master = util.get_quorum( m )
self.assertEqual( self.quorum_policy[1], quorum_of_haning_master,
'invalid quorum of haning master, expected:%d, but:%d' %( self.quorum_policy[1], quorum_of_haning_master) )
util.log( 'succeeded : quorum of haning master=%d' % quorum_of_haning_master )
# check quorum policy
quorum_of_new_master = util.get_quorum( new_master )
self.assertNotEqual( None, quorum_of_new_master, 'failed : find new master' )
self.assertEqual( self.quorum_policy[1], quorum_of_new_master ,
'invalid quorum of new master, expected:%d, but:%d' % (self.quorum_policy[1], quorum_of_new_master) )
util.log( 'succeeded : quorum of new master=%d' % quorum_of_new_master )
return 0
def request_to_shutdown_cm(server):
id = server['id']
if util.shutdown_cm(id) is not 0:
util.log('failed to shutdown_cm%d' % (id))
return -1
return 0
def request_to_shutdown_cm( server ):
id = server['id']
if util.shutdown_cm( id ) is not 0:
util.log('failed to shutdown_cm%d' % (id))
return -1
return 0
