def setUp(self):
util.set_process_logfile_prefix( 'TestBasicOp_%s' % self._testMethodName )
ret = default_cluster.initialize_starting_up_smr_before_redis(self.cluster)
if ret is not 0:
util.log('failed to test_basic_op.initialize')
default_cluster.finalize(self.cluster)
self.assertEquals( ret, 0, 'failed to test_basic_op.initialize' )
def setUp(self):
util.set_process_logfile_prefix('TestScan_%s' % self._testMethodName)
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(self.cluster)
if self.conf_checker is None:
util.log('failed to TestScan.initialize')
return -1
return 0
def setUp(self):
util.set_process_logfile_prefix( 'TestBasicOp_%s' % self._testMethodName )
ret = default_cluster.initialize_starting_up_smr_before_redis(self.cluster)
if ret is not 0:
util.log('failed to test_basic_op.initialize')
default_cluster.finalize(self.cluster)
self.assertEquals( ret, 0, 'failed to test_basic_op.initialize' )
def setUp(self):
util.set_process_logfile_prefix('TestScaleout_%s' %
self._testMethodName)
conf = {'smr_log_delete_delay': 60}
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster, conf=conf)
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUpClass(cls):
ret = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster)
if ret is not 0:
default_cluster.finalize(cls.cluster)
assert (ret == 0)
return 0
def setUp(self):
util.set_process_logfile_prefix('TestClusterUtil_%s' %
self._testMethodName)
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
return 0
def setUp(self):
util.set_process_logfile_prefix( 'TestScaleout_%s' % self._testMethodName )
conf = {'smr_log_delete_delay':60}
if default_cluster.initialize_starting_up_smr_before_redis( self.cluster, conf=conf ) is not 0:
util.log('failed to TestScaleout.initialize')
return -1
return 0
def main():
if len(sys.argv) < 2:
print USAGE
return -1
signal.signal(signal.SIGINT, signal_handler)
# Verify config
config.verify_config()
# Init options
try:
opts, args = getopt.getopt(
sys.argv[1:],
"inl:sb",
["init", "non-interactive", "backup_log_dir", "skip-copy_binaries", "32bit-binary-test"],
)
except getopt.GetoptError as e:
print USAGE
print e
return -1
opt_init = False
opt_backup_log_dir = None
opt_skip_copy_binaries = False
opt_32bit_binary_test = False
opt_non_interactive = False
for opt, arg in opts:
if opt in ("-i", "--init"):
opt_init = True
elif opt in ("-l", "--backup_log_dir"):
opt_backup_log_dir = arg
elif opt in ("-s", "--skip-copy-binareis"):
opt_skip_copy_binaries = True
elif opt in ("-b", "--32bit-binary-test"):
opt_32bit_binary_test = True
elif opt in ("-n", "--non-interactive"):
opt_non_interactive = True
# Clean up test environment
if cleanup_test_env(opt_skip_copy_binaries, opt_32bit_binary_test) != 0:
print "Clean up test environment fail! Aborting..."
return -1
# When -i flag is on, it exits after setting up a cluster.
if opt_init is True:
if default_cluster.initialize_starting_up_smr_before_redis(config.clusters[0], verbose=2) is not 0:
util.log("failed setting up servers.")
else:
util.log("finished successfully setting up servers.")
return 0
# Load test modules
module_list = load_test_modules(opt_32bit_binary_test)
print "module list : "
print module_list
# Run test
return test_modules(module_list, opt_non_interactive, opt_backup_log_dir)
def setup_test_cluster( self, cluster ):
self.cluster = cluster
self.leader_cm = cluster['servers'][0]
self.mgmt_ip = self.leader_cm['ip']
self.mgmt_port = self.leader_cm['cm_port']
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster )
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUp(self):
util.set_process_logfile_prefix('TestGatewayMgmt_%s' %
self._testMethodName)
if default_cluster.initialize_starting_up_smr_before_redis(
self.cluster) is not 0:
util.log('failed to TestScaleout.initialize')
return -1
return 0
def setUp(self):
util.set_process_logfile_prefix('TestFreeClient_%s' %
self._testMethodName)
ret = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
if ret is not 0:
default_cluster.finalize(self.cluster)
return 0
def setUp(self):
util.set_process_logfile_prefix('TestScan_%s' % self._testMethodName)
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
if self.conf_checker is None:
util.log('failed to TestScan.initialize')
return -1
return 0
def setUp(self):
util.set_process_logfile_prefix('TestLocalProxy_%s' %
self._testMethodName)
ret = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
if ret is not 0:
util.log('failed to test_local_proxy.initialize')
default_cluster.finalize(self.cluster)
def setUp(self):
util.set_process_logfile_prefix('TestHeartbeatChecker_%s' %
self._testMethodName)
ret = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
if ret is not 0:
default_cluster.finalize(self.cluster)
self.assertEquals(ret, 0, 'failed to TestHeartbeatChecker.initialize')
def setup_test_cluster(self, cluster):
self.cluster = cluster
self.leader_cm = cluster['servers'][0]
self.mgmt_ip = self.leader_cm['ip']
self.mgmt_port = self.leader_cm['cm_port']
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUp(self):
util.set_process_logfile_prefix('TestScaleout_%s' %
self._testMethodName)
conf = {'smr_log_delete_delay': 60}
if default_cluster.initialize_starting_up_smr_before_redis(
self.cluster, conf=conf) is not 0:
util.log('failed to TestScaleout.initialize')
return -1
return 0
def setUpClass( cls ):
cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster )
assert cls.conf_checker != None, 'failed to initialize cluster'
slave = util.get_server_by_role( cls.cluster['servers'], 'slave' )
for i in range( cls.max_load_generator ):
cls.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, slave['ip'], slave['redis_port'] )
cls.load_gen_thrd_list[i].start()
def setUpClass(cls):
ret = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster)
if ret is not 0:
util.log(
'failed to initialize_starting_up_smr_before_redis in TestUpgrade'
)
default_cluster.finalize(cls.cluster)
return 0
def setUpClass(cls):
global mgmt_ip
global mgmt_port
mgmt_ip = cls.leader_cm['ip']
mgmt_port = cls.leader_cm['cm_port']
cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster)
assert cls.conf_checker != None, 'failed to initialize cluster'
def main():
if len(sys.argv) < 2:
print USAGE
return -1
signal.signal( signal.SIGINT, signal_handler )
# Verify config
config.verify_config()
# Init options
try:
opts, args = getopt.getopt(sys.argv[1:], 'inl:sb', ['init', 'non-interactive', 'backup_log_dir', 'skip-copy_binaries', '32bit-binary-test'])
except getopt.GetoptError as e:
print USAGE
print e
return -1
opt_init = False
opt_backup_log_dir = None
opt_skip_copy_binaries = False
opt_32bit_binary_test = False
opt_non_interactive = False
for opt, arg in opts:
if opt in ("-i", '--init'):
opt_init = True
elif opt in ("-l", '--backup_log_dir'):
opt_backup_log_dir = arg
elif opt in ("-s", '--skip-copy-binareis'):
opt_skip_copy_binaries = True
elif opt in ("-b", '--32bit-binary-test'):
opt_32bit_binary_test = True
elif opt in ("-n", '--non-interactive'):
opt_non_interactive = True
# Clean up test environment
if cleanup_test_env(opt_skip_copy_binaries, opt_32bit_binary_test) != 0:
print 'Clean up test environment fail! Aborting...'
return -1
# When -i flag is on, it exits after setting up a cluster.
if opt_init is True:
if default_cluster.initialize_starting_up_smr_before_redis( config.clusters[0], verbose=2 ) is None:
util.log('failed setting up servers.')
else:
util.log('finished successfully setting up servers.' )
return 0
# Load test modules
module_list = load_test_modules(opt_32bit_binary_test)
print "module list : "
print module_list
# Run test
return test_modules(module_list, opt_non_interactive, opt_backup_log_dir)
def setUpClass(cls):
cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster)
assert cls.conf_checker != None, 'failed to initialize cluster'
slave = util.get_server_by_role(cls.cluster['servers'], 'slave')
for i in range(cls.max_load_generator):
cls.load_gen_thrd_list[i] = load_generator.LoadGenerator(
i, slave['ip'], slave['redis_port'])
cls.load_gen_thrd_list[i].start()
def setUp(self):
# Initialize cluster
util.set_process_logfile_prefix( 'TestARCCI%s' % self._testMethodName )
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(self.cluster, arch=self.arch)
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
self.GW_LIST = []
for s in self.cluster['servers']:
self.GW_LIST.append({"ip":s['ip'],"port":s['gateway_port']})
return 0
def setUpClass( cls ):
ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster )
if ret is not 0:
default_cluster.finalize( cls.cluster )
return -1
slave = util.get_server_by_role( cls.cluster['servers'], 'slave' )
for i in range( cls.max_load_generator ):
cls.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, slave['ip'], slave['redis_port'] )
cls.load_gen_thrd_list[i].start()
def setUp(self):
util.set_process_logfile_prefix('TestClusterUtil_%s' % self._testMethodName)
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(self.cluster)
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
# Setup proxy server
self.getdump_proxy_proc = self.setup_proxy_server(self.getdump_proxy_port, self.cluster['servers'][0]['redis_port'],
"/dev/null", self.getdump_proxy_log, "getdump.fifo")
self.playdump_proxy_proc = self.setup_proxy_server(self.playdump_proxy_port, self.cluster['servers'][0]['redis_port'],
self.playdump_proxy_log, "/dev/null", "playdump.fifo")
return 0
def setup_test_cluster( self, cluster ):
self.cluster = cluster
self.leader_cm = cluster['servers'][0]
self.mgmt_ip = self.leader_cm['ip']
self.mgmt_port = self.leader_cm['cm_port']
ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster )
if ret is not 0:
default_cluster.finalize( self.cluster )
self.assertEquals( ret, 0, 'failed to TestPGSHanging.initialize' )
return 0
def setup_test_cluster(self, cluster):
self.cluster = cluster
self.leader_cm = cluster['servers'][0]
self.mgmt_ip = self.leader_cm['ip']
self.mgmt_port = self.leader_cm['cm_port']
ret = default_cluster.initialize_starting_up_smr_before_redis(
self.cluster)
if ret is not 0:
default_cluster.finalize(self.cluster)
self.assertEquals(ret, 0, 'failed to TestPGSHanging.initialize')
return 0
def setUp(self):
util.set_process_logfile_prefix('TestClusterUtil_%s' % self._testMethodName)
if default_cluster.initialize_starting_up_smr_before_redis(self.cluster) is not 0:
util.log('failed to TestClusterUtil.initialize')
return -1
# Setup proxy server
self.getdump_proxy_proc = self.setup_proxy_server(self.getdump_proxy_port, self.cluster['servers'][0]['redis_port'],
"/dev/null", self.getdump_proxy_log, "getdump.fifo")
self.playdump_proxy_proc = self.setup_proxy_server(self.playdump_proxy_port, self.cluster['servers'][0]['redis_port'],
self.playdump_proxy_log, "/dev/null", "playdump.fifo")
return 0
def setUpClass( cls ):
global mgmt_ip
global mgmt_port
mgmt_ip = cls.leader_cm['ip']
mgmt_port = cls.leader_cm['cm_port']
ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster )
if ret is not 0:
util.log( 'failed to initialize_starting_up_smr_before_redis in TestUpgrade' )
default_cluster.finalize( cls.cluster )
return 0
def setUpClass(cls):
ret = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster)
if ret is not 0:
default_cluster.finalize(cls.cluster)
return -1
slave = util.get_server_by_role(cls.cluster['servers'], 'slave')
for i in range(cls.max_load_generator):
cls.load_gen_thrd_list[i] = load_generator.LoadGenerator(
i, slave['ip'], slave['redis_port'])
cls.load_gen_thrd_list[i].start()
def setUp(self):
# Initialize cluster
util.set_process_logfile_prefix( 'TestARCCI%s' % self._testMethodName )
ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster )
if ret is not 0:
default_cluster.finalize( self.cluster )
self.assertEquals( ret, 0, 'failed to test_arcci.initialize' )
self.GW_LIST = []
for s in self.cluster['servers']:
self.GW_LIST.append({"ip":s['ip'],"port":s['gateway_port']})
return 0
def setUpClass(cls):
global mgmt_ip
global mgmt_port
mgmt_ip = cls.leader_cm['ip']
mgmt_port = cls.leader_cm['cm_port']
ret = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster)
if ret is not 0:
util.log(
'failed to initialize_starting_up_smr_before_redis in TestUpgrade'
)
default_cluster.finalize(cls.cluster)
return 0
def setUpClass(cls):
cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(
cls.cluster, verbose=2)
assert cls.conf_checker != None, 'failed to initialize cluster'
def setUp( self ):
util.set_process_logfile_prefix( 'TestQuorumPolicy_%s' % self._testMethodName )
ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster )
if ret is not 0:
default_cluster.finalize( self.cluster )
self.assertEquals( ret, 0, 'failed to TestQuorumPolicy.initialize' )
def setUpClass( cls ):
cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster )
assert cls.conf_checker != None, 'failed to initialize cluster'
def setUpClass( cls ):
ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster )
if ret is not 0:
util.log( 'failed to initialize_starting_up_smr_before_redis in TestUpgrade' )
default_cluster.finalize( cls.cluster )
return 0
def setUp(self):
util.set_process_logfile_prefix('TestLocalProxy_%s' % self._testMethodName)
ret = default_cluster.initialize_starting_up_smr_before_redis(self.cluster)
if ret is not 0:
util.log('failed to test_local_proxy.initialize')
default_cluster.finalize(self.cluster)
def test_5_mgmt_is_isolated_with_lconn(self):
util.print_frame()
util.iptables_print_list()
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Set SMR option (slave_idle_timeout)
util.log('\n\n\n ### Set SMR option ###')
for s in cluster['servers']:
t = telnet.Telnet('SMR%d' % s['id'])
self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0,
'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port']))
cmd = 'confset slave_idle_timeout_msec 18000'
util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd))
t.write('confset slave_idle_timeout_msec 18000\r\n')
reply = t.read_until('\r\n').strip()
util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply))
self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply)
# Network isolation test
for loop_cnt in range(3):
# Get master
master = util.get_server_by_role_and_pg( cluster['servers'], 'master', 0 )
first_slave = None
for s in cluster['servers']:
if s == master:
continue
# Skip non-virtual host
if s.has_key('real_ip') == False:
continue
if first_slave == None:
first_slave = s
# 'role lconn'
util.log( 'role lconn pgs%d while hanging.' % s['id'] )
ret = util.role_lconn_addr( s['real_ip'], s['smr_mgmt_port'] )
self.assertEqual( ret, '+OK\r\n', 'role lconn failed. reply="%s"' % (ret[:-2]) )
util.log( 'succeeded : cmd="role lconn", reply="%s"' % (ret[:-2]) )
time.sleep(0.5)
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt)
self.assertTrue(util.iptables_drop('A', '127.0.0.100', first_slave['smr_mgmt_port']), 'add a bloking role to iptables fail.')
for i in range(6):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(10):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['pgs_id'] != first_slave['id']:
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
self.assertTrue(util.iptables_drop('D', '127.0.0.100', first_slave['smr_mgmt_port']), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(7):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == 1:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
ok = False
for i in xrange(5):
ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True)
if ok:
break
else:
time.sleep(1)
self.assertTrue(ok, 'failed to check cluster state')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(len(final_state)):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def test_4_mgmt_is_isolated_with_red_failover(self):
util.print_frame()
util.iptables_print_list()
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port)
# Master must be the first pgs, cluster['servers'][0].
to_be_master = cluster['servers'][0]
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', to_be_master['pg_id'])
master_id = -1
if m['id'] != to_be_master['id']:
try_cnt = 0
while master_id != to_be_master['id'] and try_cnt < 20:
master_id = util.role_change(cluster['servers'][0], cluster['cluster_name'], to_be_master['id'])
try_cnt += 1
time.sleep(1)
self.assertEquals(master_id, to_be_master['id'], 'change %d to a master fail' % to_be_master['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Set SMR option (slave_idle_timeout)
util.log('\n\n\n ### Set SMR option ###')
for s in cluster['servers']:
t = telnet.Telnet('SMR%d' % s['id'])
self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0,
'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port']))
cmd = 'confset slave_idle_timeout_msec 18000'
util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd))
t.write('confset slave_idle_timeout_msec 18000\r\n')
reply = t.read_until('\r\n').strip()
util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply))
self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply)
# Network isolation test
for loop_cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt)
for s in cluster['servers']:
self.assertTrue(util.iptables_drop('A', '127.0.0.100', s['smr_mgmt_port']), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['ip'] != '127.0.0.100':
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
pgs_list = util.get_pgs_info_list(mgmt_ip, mgmt_port, cluster)
reds = filter(lambda x: x['color'] == 'RED', pgs_list)
# Shutdown
server = cluster['servers'][random.choice(reds)['pgs_id']]
util.log( 'shutdown pgs%d while hanging.' % server['id'] )
ret = testbase.request_to_shutdown_smr( server )
self.assertEqual( ret, 0, 'failed to shutdown smr. id:%d' % server['id'] )
ret = testbase.request_to_shutdown_redis( server )
self.assertEqual( ret, 0, 'failed to shutdown redis. id:%d' % server['id'] )
# Check state F
max_try = 20
expected = 'F'
for i in range( 0, max_try):
util.log('MGMT_IP:%s, MGMT_PORT:%d' % (mgmt_ip, mgmt_port))
state = util._get_smr_state( server['id'], cluster['cluster_name'], mgmt_ip, mgmt_port )
if expected == state:
break;
time.sleep( 1 )
self.assertEqual( expected , state,
'server%d - state:%s, expected:%s' % (server['id'], state, expected) )
util.log( 'succeeded : pgs%d state changed to F.' % server['id'] )
# Unblock network
for s in cluster['servers']:
self.assertTrue(util.iptables_drop('D', '127.0.0.100', s['smr_mgmt_port']), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(10):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == server['id']:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Recovery
util.log( 'restart pgs%d.' % server['id'] )
ret = testbase.request_to_start_smr( server )
self.assertEqual( ret, 0, 'failed to start smr. id:%d' % server['id'] )
ret = testbase.request_to_start_redis( server )
self.assertEqual( ret, 0, 'failed to start redis. id:%d' % server['id'] )
wait_count = 20
ret = testbase.wait_until_finished_to_set_up_role( server, wait_count )
self.assertEqual( ret, 0, 'failed to role change. smr_id:%d' % (server['id']) )
redis = redis_mgmt.Redis( server['id'] )
ret = redis.connect( server['ip'], server['redis_port'] )
self.assertEqual( ret, 0, 'failed to connect to redis' )
ok = False
for i in xrange(5):
ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True)
if ok:
break
else:
time.sleep(1)
self.assertTrue(ok, 'failed to check cluster state')
# Reset SMR option (slave_idle_timeout)
t = telnet.Telnet('SMR%d' % server['id'])
self.assertEqual(t.connect(server['ip'], server['smr_mgmt_port']), 0,
'Failed to connect to smr. ADDR=%s:%d' % (server['ip'], server['smr_mgmt_port']))
cmd = 'confset slave_idle_timeout_msec 18000'
util.log('[%s:%d] >> %s' % (server['ip'], server['smr_mgmt_port'], cmd))
t.write('confset slave_idle_timeout_msec 18000\r\n')
reply = t.read_until('\r\n').strip()
util.log('[%s:%d] << %s' % (server['ip'], server['smr_mgmt_port'], reply))
self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply)
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(len(final_state)):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
if initial_state[i]['pgs_id'] == 1:
self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def setUp(self):
util.set_process_logfile_prefix( 'TestFreeClient_%s' % self._testMethodName )
ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster )
if ret is not 0:
default_cluster.finalize( self.cluster )
return 0
def test_6_repeat_isolation_and_no_opinion_linepay(self):
util.print_frame()
out = util.sudo('iptables -L')
util.log('====================================================================')
util.log('out : %s' % out)
util.log('out.return_code : %d' % out.return_code)
util.log('out.stderr : %s' % out.stderr)
util.log('out.succeeded : %s' % out.succeeded)
# Add forwarding role
out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
cluster_name = 'no_opinion'
cluster = filter(lambda x: x['cluster_name'] == cluster_name, config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
ret = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertEqual(0, ret, 'failed to TestMaintenance.initialize')
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
loop_cnt = 0
while (loop_cnt < 20):
loop_cnt += 1
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt)
out = util.sudo('iptables -A OUTPUT -d 127.0.0.102 -j DROP')
self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)
for i in range(1):
util.log('waiting... %d' % (i + 1))
time.sleep(0.1)
out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)
for i in range(3):
util.log('waiting... %d' % (i + 1))
time.sleep(1.2)
out = util.sudo('iptables -A OUTPUT -d 127.0.0.101 -j DROP')
self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)
for i in range(1):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % loop_cnt)
out = util.sudo('iptables -D OUTPUT -d 127.0.0.102 -j DROP')
self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)
for i in range(0):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)
for i in range(0):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
out = util.sudo('iptables -D OUTPUT -d 127.0.0.101 -j DROP')
self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)
for i in range(3):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Print state of cluster
util.log('\n ### STATE OF CLUSTER ### ')
cluster_state = False
for i in range(10):
cluster_state = util.check_cluster(cluster_name, mgmt_ip, mgmt_port, initial_state, check_quorum=True)
if cluster_state == True:
break
else:
time.sleep(1)
self.assertTrue(cluster_state, 'failed to check cluster state')
all_in_f = True
for s in cluster['servers']:
if checkLastState(mgmt_ip, s['cm_port'], cluster_name, 0, 'F') == False:
all_in_f = False
break
self.assertFalse(all_in_f, 'PGS0`s last state remains in F')
# Shutdown cluster
ret = default_cluster.finalize( cluster )
self.assertEqual(ret, 0, 'failed to TestMaintenance.finalize')
# Delete forwarding role
out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
def setUp(self):
util.set_process_logfile_prefix("TestGatewayMgmt_%s" % self._testMethodName)
if default_cluster.initialize_starting_up_smr_before_redis(self.cluster) is not 0:
util.log("failed to TestScaleout.initialize")
return -1
return 0
def setUp(self):
util.set_process_logfile_prefix( 'TestScaleout_%s' % self._testMethodName )
conf = {'smr_log_delete_delay':60}
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster, conf=conf )
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUp( self ):
util.set_process_logfile_prefix( 'TestQuorumPolicy_%s' % self._testMethodName )
self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster )
self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUpClass( cls ):
ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster )
if ret is not 0:
default_cluster.finalize( cls.cluster )
assert (ret == 0)
return 0
def test_6_repeat_isolation_and_no_opinion_linepay(self):
util.print_frame()
util.iptables_print_list()
# Add forwarding role
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
self.assertTrue(util.iptables_redirect('A', '127.0.0.101', '127.0.0.1'), 'add a forwarding role to iptables fail.')
self.assertTrue(util.iptables_redirect('A', '127.0.0.102', '127.0.0.1'), 'add a forwarding role to iptables fail.')
cluster_name = 'no_opinion'
cluster = filter(lambda x: x['cluster_name'] == cluster_name, config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
loop_cnt = 0
while (loop_cnt < 20):
loop_cnt += 1
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt)
self.assertTrue(util.iptables_drop('A', '127.0.0.102'), 'add a bloking role to iptables fail.')
for i in range(1):
util.log('waiting... %d' % (i + 1))
time.sleep(0.1)
self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')
for i in range(3):
util.log('waiting... %d' % (i + 1))
time.sleep(1.2)
self.assertTrue(util.iptables_drop('A', '127.0.0.101'), 'add a bloking role to iptables fail.')
for i in range(1):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % loop_cnt)
self.assertTrue(util.iptables_drop('D', '127.0.0.102'), 'delete a bloking role to iptables fail.')
for i in range(0):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')
for i in range(0):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
self.assertTrue(util.iptables_drop('D', '127.0.0.101'), 'delete a bloking role to iptables fail.')
for i in range(3):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Print state of cluster
util.log('\n ### STATE OF CLUSTER ### ')
cluster_state = False
for i in range(10):
cluster_state = util.check_cluster(cluster_name, mgmt_ip, mgmt_port, initial_state, check_quorum=True)
if cluster_state == True:
break
else:
time.sleep(1)
self.assertTrue(cluster_state, 'failed to check cluster state')
all_in_f = True
for s in cluster['servers']:
if checkLastState(mgmt_ip, s['cm_port'], cluster_name, 0, 'F') == False:
all_in_f = False
break
self.assertFalse(all_in_f, 'PGS0`s last state remains in F')
# Delete forwarding role
self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail.')
self.assertTrue(util.iptables_redirect('D', '127.0.0.101', '127.0.0.1'), 'delete a forwarding role to iptables fail.')
self.assertTrue(util.iptables_redirect('D', '127.0.0.102', '127.0.0.1'), 'delete a forwarding role to iptables fail.')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def test_7_dirty_network_fi(self):
util.print_frame()
clnts = []
try:
util.iptables_print_list()
# Add forwarding role
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
cluster_name = 'network_isolation_cluster_1'
cluster = filter(lambda x: x['cluster_name'] == cluster_name, config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster,
conf={'cm_context':'applicationContext-fi.xml'})
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Start crc16 client
for s in cluster['servers']:
c = load_generator_crc16.Crc16Client(s['id'], s['ip'], s['redis_port'], 600, verbose=False)
c.start()
clnts.append(c)
# Network isolation test
cmfi = fi_confmaster.ConfmasterWfFi(['ra', 'me', 'yj', 'bj', 'mg'],
['lconn', 'slave', 'master', 'setquorum'], [True, False], 1)
for fi in cmfi:
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %s ### ' % str(fi))
ret = block_network(cluster, mgmt_ip, mgmt_port)
self.assertTrue(ret, '[%s] failed to block network.' % str(fi))
for i in xrange(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in xrange(10):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
state_transition_done = True
for s in isolated_states:
if s['ip'] != '127.0.0.100':
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Fault injection
try:
self.assertTrue(fi_confmaster.fi_add(fi, 1, mgmt_ip, mgmt_port),
"Confmaster command fail. fi: %s" % str(fi))
except ValueError as e:
self.fail("Confmaster command error. cmd: \"%s\", reply: \"%s\"" % (cmd, reply))
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %s ### ' % str(fi))
ret = unblock_network(cluster, mgmt_ip, mgmt_port, None)
self.assertTrue(ret, '[%s] failed to unblock network.' % str(fi))
for i in xrange(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in xrange(10):
isolated_states = []
ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
if ok:
break
time.sleep(1)
self.assertTrue(ok, '[%s] Fail. unstable cluster.' % str(fi))
check_cluster = False
# 'bj', 'slave'
if fi[0] == 'bj' and fi[1] == 'slave':
m, s1, s2 = util.get_mss(cluster)
ret = util.role_lconn(s1)
self.assertEqual("+OK\r\n", ret, '[%s] role lconn fail.' % str(fi))
check_cluster = True
# 'me', 'lconn'
elif fi[0] == 'me':
m, s1, s2 = util.get_mss(cluster)
ret = util.role_lconn(m)
self.assertEqual("+OK\r\n", ret, '[%s] role lconn fail.' % str(fi))
check_cluster = True
# 'setquorum'
elif fi[1] == 'setquorum':
m, s1, s2 = util.get_mss(cluster)
ret = util.cmd_to_smr_addr(s1['ip'], s1['smr_mgmt_port'], 'fi delay sleep 1 8000\r\n', timeout=20)
self.assertEqual("+OK\r\n", ret, '[%s] "fi delay sleep 1 8000" fail. ret: "%s"' % (str(fi), ret))
check_cluster = True
if check_cluster:
# Check cluster state
ok = False
for i in xrange(20):
isolated_states = []
ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
if ok:
break
time.sleep(1)
self.assertTrue(ok, '[%s] Fail. unstable cluster.' % str(fi))
# Check fault injection
ok = False
for i in xrange(10):
count = fi_confmaster.fi_count(fi, mgmt_ip, mgmt_port)
if count == 0:
ok = True
break
time.sleep(0.5)
self.assertTrue(ok, "[%s] fail. failt injection had not been triggered." % str(fi))
for c in clnts:
self.assertTrue(c.is_consistency(), '[%s] data consistency error!' % str(fi))
for c in clnts:
self.assertTrue(c.is_consistency(), '[%s] data consistency error!' % str(fi))
# Go back to initial configuration
cmfi.init()
for fi in cmfi:
try:
self.assertTrue(fi_confmaster.fi_add(fi, 0, mgmt_ip, mgmt_port),
"Confmaster command fail. fi: %s" % str(fi))
except ValueError as e:
self.fail("Confmaster command error. cmd: \"%s\", reply: \"%s\"" % (cmd, reply))
# Wait until workflows done
ret = util.await(60, True)(
lambda cinfo: cinfo['wf'] == 0,
lambda : util.cluster_info(mgmt_ip, mgmt_port, cluster['cluster_name']))
self.assertTrue(ret, 'There are still some workflows.')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
finally:
for c in clnts:
c.quit()
for c in clnts:
c.join()
# Delete forwarding role
self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
def test_1_mgmt_is_isolated(self):
util.print_frame()
util.iptables_print_list()
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0]
util.log(util.json_to_str(cluster))
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Set SMR option (slave_idle_timeout)
util.log('\n\n\n ### Set SMR option ###')
for s in cluster['servers']:
t = telnet.Telnet('SMR%d' % s['id'])
self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0,
'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port']))
cmd = 'confset slave_idle_timeout_msec 18000'
util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd))
t.write('confset slave_idle_timeout_msec 18000\r\n')
reply = t.read_until('\r\n').strip()
util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply))
self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply)
# Network isolation test
for cnt in range(5):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt)
for s in cluster['servers']:
"""Loopback Address Range (Reference : RFC3330)
127.0.0.0/8 - This block is assigned for use as the Internet host
loopback address. A datagram sent by a higher level protocol to an
address anywhere within this block should loop back inside the host.
This is ordinarily implemented using only 127.0.0.1/32 for loopback,
but no addresses within this block should ever appear on any network
anywhere [RFC1700, page 5].
"""
self.assertTrue(util.iptables_drop('A', '127.0.0.100', s['smr_mgmt_port']), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['ip'] != '127.0.0.100':
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt)
for s in cluster['servers']:
self.assertTrue(util.iptables_drop('D', '127.0.0.100', s['smr_mgmt_port']), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(7):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
all_green = True
for s in final_state:
if is_pgs_normal(s) == False:
all_green = False
if all_green:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def test_2_some_pgs_is_isolated(self):
util.print_frame()
util.iptables_print_list()
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_2', config.clusters)[0]
util.log(util.json_to_str(cluster))
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Place master on virtual ip address in order to cause master election.
pg_id = 0
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id)
s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id)
if m.has_key('ip') == True and m.has_key('real_ip') == False:
ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id'])
self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
for cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['pgs_id'] == 1:
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(7):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == 1:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(len(final_state)):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
# Delete forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def setUp(self):
util.set_process_logfile_prefix( 'TestMaintenance_%s' % self._testMethodName )
if default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) is not 0:
util.log('failed to TestMaintenance.initialize')
return -1
return 0
def test_5_mgmt_is_isolated_with_master_failover(self):
util.print_frame()
out = util.sudo('iptables -L')
util.log('====================================================================')
util.log('out : %s' % out)
util.log('out.return_code : %d' % out.return_code)
util.log('out.stderr : %s' % out.stderr)
util.log('out.succeeded : %s' % out.succeeded)
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
ret = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertEqual(0, ret, 'failed to TestMaintenance.initialize')
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
for loop_cnt in range(3):
master, slave1, slave2 = util.get_mss(cluster)
self.assertNotEquals(master, None, 'there is no master')
self.assertNotEquals(slave1, None, 'there is no slave1')
self.assertNotEquals(slave2, None, 'there is no slave2')
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt)
out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(10):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['ip'] != '127.0.0.100':
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Shutdown master
util.log( 'shutdown pgs%d while hanging.' % master['id'] )
ret = testbase.request_to_shutdown_smr( master )
self.assertEqual( ret, 0, 'failed to shutdown smr. id:%d' % master['id'] )
ret = testbase.request_to_shutdown_redis( master )
self.assertEqual( ret, 0, 'failed to shutdown redis. id:%d' % master['id'] )
# Check state F
max_try = 20
expected = 'F'
for i in range( 0, max_try):
util.log('MGMT_IP:%s, MGMT_PORT:%d' % (mgmt_ip, mgmt_port))
state = util._get_smr_state( master['id'], cluster['cluster_name'], mgmt_ip, mgmt_port )
if expected == state:
break;
time.sleep( 1 )
self.assertEqual( expected , state,
'master%d - state:%s, expected:%s' % (master['id'], state, expected) )
util.log( 'succeeded : pgs%d state changed to F.' % master['id'] )
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % loop_cnt)
out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)
# Check cluster state
ok = False
for i in range(7):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == master['id']:
continue
if s['active_role'] != s['mgmt_role']:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Recovery
util.log( 'restart pgs%d.' % master['id'] )
ret = testbase.request_to_start_smr( master )
self.assertEqual( ret, 0, 'failed to start smr. id:%d' % master['id'] )
ret = testbase.request_to_start_redis( master )
self.assertEqual( ret, 0, 'failed to start redis. id:%d' % master['id'] )
wait_count = 20
ret = testbase.wait_until_finished_to_set_up_role( master, wait_count )
self.assertEqual( ret, 0, 'failed to role change. smr_id:%d' % (master['id']) )
redis = redis_mgmt.Redis( master['id'] )
ret = redis.connect( master['ip'], master['redis_port'] )
self.assertEqual( ret, 0, 'failed to connect to redis' )
ok = False
for i in xrange(5):
ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True)
if ok:
break
else:
time.sleep(1)
self.assertTrue(ok, 'failed to check cluster state')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(0, 3):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
for i in range(3, 6):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
self.assertEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state')
# Shutdown cluster
ret = default_cluster.finalize( cluster )
self.assertEqual(ret, 0, 'failed to TestMaintenance.finalize')
# Delete forwarding role (127.0.0.100 -> 127.0.0.1)
out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
def test_3_some_pgs_is_isolated_2copy(self):
util.print_frame()
out = util.sudo('iptables -L')
util.log('====================================================================')
util.log('out : %s' % out)
util.log('out.return_code : %d' % out.return_code)
util.log('out.stderr : %s' % out.stderr)
util.log('out.succeeded : %s' % out.succeeded)
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1_2copy', config.clusters)[0]
util.log(util.json_to_str(cluster))
# MGMT
mgmt_ip = cluster['servers'][0]['ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Place master on real ip address
for pg_id in [0, 1]:
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id)
s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id)
if m.has_key('ip') and m.has_key('real_ip'):
if m['ip'] != m['real_ip']:
ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id'])
self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
for cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['pgs_id'] == 0 or s['pgs_id'] == 1:
continue
if s['active_role'] != 'M' or s['mgmt_role'] != 'M':
state_transition_done = False
if s['quorum'] != 0:
state_transition_done = False
if state_transition_done:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(7):
final_state = []
if util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) == False:
time.sleep(1)
continue
state_consistency = True
for s in final_state:
if s['pgs_id'] == 1:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
# Delete forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail.')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
