def setUp(self): util.set_process_logfile_prefix( 'TestBasicOp_%s' % self._testMethodName ) ret = default_cluster.initialize_starting_up_smr_before_redis(self.cluster) if ret is not 0: util.log('failed to test_basic_op.initialize') default_cluster.finalize(self.cluster) self.assertEquals( ret, 0, 'failed to test_basic_op.initialize' )
def setUp(self): util.set_process_logfile_prefix('TestScan_%s' % self._testMethodName) self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(self.cluster) if self.conf_checker is None: util.log('failed to TestScan.initialize') return -1 return 0
def setUp(self): util.set_process_logfile_prefix( 'TestBasicOp_%s' % self._testMethodName ) ret = default_cluster.initialize_starting_up_smr_before_redis(self.cluster) if ret is not 0: util.log('failed to test_basic_op.initialize') default_cluster.finalize(self.cluster) self.assertEquals( ret, 0, 'failed to test_basic_op.initialize' )
def setUp(self): util.set_process_logfile_prefix('TestScaleout_%s' % self._testMethodName) conf = {'smr_log_delete_delay': 60} self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster, conf=conf) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUpClass(cls): ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster) if ret is not 0: default_cluster.finalize(cls.cluster) assert (ret == 0) return 0
def setUp(self): util.set_process_logfile_prefix('TestClusterUtil_%s' % self._testMethodName) self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster') return 0
def setUp(self): util.set_process_logfile_prefix( 'TestScaleout_%s' % self._testMethodName ) conf = {'smr_log_delete_delay':60} if default_cluster.initialize_starting_up_smr_before_redis( self.cluster, conf=conf ) is not 0: util.log('failed to TestScaleout.initialize') return -1 return 0
def main(): if len(sys.argv) < 2: print USAGE return -1 signal.signal(signal.SIGINT, signal_handler) # Verify config config.verify_config() # Init options try: opts, args = getopt.getopt( sys.argv[1:], "inl:sb", ["init", "non-interactive", "backup_log_dir", "skip-copy_binaries", "32bit-binary-test"], ) except getopt.GetoptError as e: print USAGE print e return -1 opt_init = False opt_backup_log_dir = None opt_skip_copy_binaries = False opt_32bit_binary_test = False opt_non_interactive = False for opt, arg in opts: if opt in ("-i", "--init"): opt_init = True elif opt in ("-l", "--backup_log_dir"): opt_backup_log_dir = arg elif opt in ("-s", "--skip-copy-binareis"): opt_skip_copy_binaries = True elif opt in ("-b", "--32bit-binary-test"): opt_32bit_binary_test = True elif opt in ("-n", "--non-interactive"): opt_non_interactive = True # Clean up test environment if cleanup_test_env(opt_skip_copy_binaries, opt_32bit_binary_test) != 0: print "Clean up test environment fail! Aborting..." return -1 # When -i flag is on, it exits after setting up a cluster. if opt_init is True: if default_cluster.initialize_starting_up_smr_before_redis(config.clusters[0], verbose=2) is not 0: util.log("failed setting up servers.") else: util.log("finished successfully setting up servers.") return 0 # Load test modules module_list = load_test_modules(opt_32bit_binary_test) print "module list : " print module_list # Run test return test_modules(module_list, opt_non_interactive, opt_backup_log_dir)
def setup_test_cluster( self, cluster ): self.cluster = cluster self.leader_cm = cluster['servers'][0] self.mgmt_ip = self.leader_cm['ip'] self.mgmt_port = self.leader_cm['cm_port'] self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUp(self): util.set_process_logfile_prefix('TestGatewayMgmt_%s' % self._testMethodName) if default_cluster.initialize_starting_up_smr_before_redis( self.cluster) is not 0: util.log('failed to TestScaleout.initialize') return -1 return 0
def setUp(self): util.set_process_logfile_prefix('TestFreeClient_%s' % self._testMethodName) ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) if ret is not 0: default_cluster.finalize(self.cluster) return 0
def setUp(self): util.set_process_logfile_prefix('TestScan_%s' % self._testMethodName) self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) if self.conf_checker is None: util.log('failed to TestScan.initialize') return -1 return 0
def setUp(self): util.set_process_logfile_prefix('TestLocalProxy_%s' % self._testMethodName) ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) if ret is not 0: util.log('failed to test_local_proxy.initialize') default_cluster.finalize(self.cluster)
def setUp(self): util.set_process_logfile_prefix('TestHeartbeatChecker_%s' % self._testMethodName) ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) if ret is not 0: default_cluster.finalize(self.cluster) self.assertEquals(ret, 0, 'failed to TestHeartbeatChecker.initialize')
def setup_test_cluster(self, cluster): self.cluster = cluster self.leader_cm = cluster['servers'][0] self.mgmt_ip = self.leader_cm['ip'] self.mgmt_port = self.leader_cm['cm_port'] self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUp(self): util.set_process_logfile_prefix('TestScaleout_%s' % self._testMethodName) conf = {'smr_log_delete_delay': 60} if default_cluster.initialize_starting_up_smr_before_redis( self.cluster, conf=conf) is not 0: util.log('failed to TestScaleout.initialize') return -1 return 0
def setUpClass( cls ): cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster ) assert cls.conf_checker != None, 'failed to initialize cluster' slave = util.get_server_by_role( cls.cluster['servers'], 'slave' ) for i in range( cls.max_load_generator ): cls.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, slave['ip'], slave['redis_port'] ) cls.load_gen_thrd_list[i].start()
def setUpClass(cls): ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster) if ret is not 0: util.log( 'failed to initialize_starting_up_smr_before_redis in TestUpgrade' ) default_cluster.finalize(cls.cluster) return 0
def setUpClass(cls): global mgmt_ip global mgmt_port mgmt_ip = cls.leader_cm['ip'] mgmt_port = cls.leader_cm['cm_port'] cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster) assert cls.conf_checker != None, 'failed to initialize cluster'
def main(): if len(sys.argv) < 2: print USAGE return -1 signal.signal( signal.SIGINT, signal_handler ) # Verify config config.verify_config() # Init options try: opts, args = getopt.getopt(sys.argv[1:], 'inl:sb', ['init', 'non-interactive', 'backup_log_dir', 'skip-copy_binaries', '32bit-binary-test']) except getopt.GetoptError as e: print USAGE print e return -1 opt_init = False opt_backup_log_dir = None opt_skip_copy_binaries = False opt_32bit_binary_test = False opt_non_interactive = False for opt, arg in opts: if opt in ("-i", '--init'): opt_init = True elif opt in ("-l", '--backup_log_dir'): opt_backup_log_dir = arg elif opt in ("-s", '--skip-copy-binareis'): opt_skip_copy_binaries = True elif opt in ("-b", '--32bit-binary-test'): opt_32bit_binary_test = True elif opt in ("-n", '--non-interactive'): opt_non_interactive = True # Clean up test environment if cleanup_test_env(opt_skip_copy_binaries, opt_32bit_binary_test) != 0: print 'Clean up test environment fail! Aborting...' return -1 # When -i flag is on, it exits after setting up a cluster. if opt_init is True: if default_cluster.initialize_starting_up_smr_before_redis( config.clusters[0], verbose=2 ) is None: util.log('failed setting up servers.') else: util.log('finished successfully setting up servers.' ) return 0 # Load test modules module_list = load_test_modules(opt_32bit_binary_test) print "module list : " print module_list # Run test return test_modules(module_list, opt_non_interactive, opt_backup_log_dir)
def setUpClass(cls): cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster) assert cls.conf_checker != None, 'failed to initialize cluster' slave = util.get_server_by_role(cls.cluster['servers'], 'slave') for i in range(cls.max_load_generator): cls.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, slave['ip'], slave['redis_port']) cls.load_gen_thrd_list[i].start()
def setUp(self): # Initialize cluster util.set_process_logfile_prefix( 'TestARCCI%s' % self._testMethodName ) self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(self.cluster, arch=self.arch) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster') self.GW_LIST = [] for s in self.cluster['servers']: self.GW_LIST.append({"ip":s['ip'],"port":s['gateway_port']}) return 0
def setUpClass( cls ): ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster ) if ret is not 0: default_cluster.finalize( cls.cluster ) return -1 slave = util.get_server_by_role( cls.cluster['servers'], 'slave' ) for i in range( cls.max_load_generator ): cls.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, slave['ip'], slave['redis_port'] ) cls.load_gen_thrd_list[i].start()
def setUp(self): util.set_process_logfile_prefix('TestClusterUtil_%s' % self._testMethodName) self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis(self.cluster) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster') # Setup proxy server self.getdump_proxy_proc = self.setup_proxy_server(self.getdump_proxy_port, self.cluster['servers'][0]['redis_port'], "/dev/null", self.getdump_proxy_log, "getdump.fifo") self.playdump_proxy_proc = self.setup_proxy_server(self.playdump_proxy_port, self.cluster['servers'][0]['redis_port'], self.playdump_proxy_log, "/dev/null", "playdump.fifo") return 0
def setup_test_cluster( self, cluster ): self.cluster = cluster self.leader_cm = cluster['servers'][0] self.mgmt_ip = self.leader_cm['ip'] self.mgmt_port = self.leader_cm['cm_port'] ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) if ret is not 0: default_cluster.finalize( self.cluster ) self.assertEquals( ret, 0, 'failed to TestPGSHanging.initialize' ) return 0
def setup_test_cluster(self, cluster): self.cluster = cluster self.leader_cm = cluster['servers'][0] self.mgmt_ip = self.leader_cm['ip'] self.mgmt_port = self.leader_cm['cm_port'] ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster) if ret is not 0: default_cluster.finalize(self.cluster) self.assertEquals(ret, 0, 'failed to TestPGSHanging.initialize') return 0
def setUp(self): util.set_process_logfile_prefix('TestClusterUtil_%s' % self._testMethodName) if default_cluster.initialize_starting_up_smr_before_redis(self.cluster) is not 0: util.log('failed to TestClusterUtil.initialize') return -1 # Setup proxy server self.getdump_proxy_proc = self.setup_proxy_server(self.getdump_proxy_port, self.cluster['servers'][0]['redis_port'], "/dev/null", self.getdump_proxy_log, "getdump.fifo") self.playdump_proxy_proc = self.setup_proxy_server(self.playdump_proxy_port, self.cluster['servers'][0]['redis_port'], self.playdump_proxy_log, "/dev/null", "playdump.fifo") return 0
def setUpClass( cls ): global mgmt_ip global mgmt_port mgmt_ip = cls.leader_cm['ip'] mgmt_port = cls.leader_cm['cm_port'] ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster ) if ret is not 0: util.log( 'failed to initialize_starting_up_smr_before_redis in TestUpgrade' ) default_cluster.finalize( cls.cluster ) return 0
def setUpClass(cls): ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster) if ret is not 0: default_cluster.finalize(cls.cluster) return -1 slave = util.get_server_by_role(cls.cluster['servers'], 'slave') for i in range(cls.max_load_generator): cls.load_gen_thrd_list[i] = load_generator.LoadGenerator( i, slave['ip'], slave['redis_port']) cls.load_gen_thrd_list[i].start()
def setUp(self): # Initialize cluster util.set_process_logfile_prefix( 'TestARCCI%s' % self._testMethodName ) ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) if ret is not 0: default_cluster.finalize( self.cluster ) self.assertEquals( ret, 0, 'failed to test_arcci.initialize' ) self.GW_LIST = [] for s in self.cluster['servers']: self.GW_LIST.append({"ip":s['ip'],"port":s['gateway_port']}) return 0
def setUpClass(cls): global mgmt_ip global mgmt_port mgmt_ip = cls.leader_cm['ip'] mgmt_port = cls.leader_cm['cm_port'] ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster) if ret is not 0: util.log( 'failed to initialize_starting_up_smr_before_redis in TestUpgrade' ) default_cluster.finalize(cls.cluster) return 0
def setUpClass(cls): cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster, verbose=2) assert cls.conf_checker != None, 'failed to initialize cluster'
def setUp( self ): util.set_process_logfile_prefix( 'TestQuorumPolicy_%s' % self._testMethodName ) ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) if ret is not 0: default_cluster.finalize( self.cluster ) self.assertEquals( ret, 0, 'failed to TestQuorumPolicy.initialize' )
def setUpClass( cls ): cls.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster ) assert cls.conf_checker != None, 'failed to initialize cluster'
def setUpClass( cls ): ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster ) if ret is not 0: util.log( 'failed to initialize_starting_up_smr_before_redis in TestUpgrade' ) default_cluster.finalize( cls.cluster ) return 0
def setUp(self): util.set_process_logfile_prefix('TestLocalProxy_%s' % self._testMethodName) ret = default_cluster.initialize_starting_up_smr_before_redis(self.cluster) if ret is not 0: util.log('failed to test_local_proxy.initialize') default_cluster.finalize(self.cluster)
def test_5_mgmt_is_isolated_with_lconn(self): util.print_frame() util.iptables_print_list() cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0] util.log(util.json_to_str(cluster)) self.leader_cm = cluster['servers'][0] # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Set SMR option (slave_idle_timeout) util.log('\n\n\n ### Set SMR option ###') for s in cluster['servers']: t = telnet.Telnet('SMR%d' % s['id']) self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0, 'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port'])) cmd = 'confset slave_idle_timeout_msec 18000' util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd)) t.write('confset slave_idle_timeout_msec 18000\r\n') reply = t.read_until('\r\n').strip() util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply)) self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply) # Network isolation test for loop_cnt in range(3): # Get master master = util.get_server_by_role_and_pg( cluster['servers'], 'master', 0 ) first_slave = None for s in cluster['servers']: if s == master: continue # Skip non-virtual host if s.has_key('real_ip') == False: continue if first_slave == None: first_slave = s # 'role lconn' util.log( 'role lconn pgs%d while hanging.' % s['id'] ) ret = util.role_lconn_addr( s['real_ip'], s['smr_mgmt_port'] ) self.assertEqual( ret, '+OK\r\n', 'role lconn failed. reply="%s"' % (ret[:-2]) ) util.log( 'succeeded : cmd="role lconn", reply="%s"' % (ret[:-2]) ) time.sleep(0.5) # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt) self.assertTrue(util.iptables_drop('A', '127.0.0.100', first_slave['smr_mgmt_port']), 'add a bloking role to iptables fail.') for i in range(6): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in range(10): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) time.sleep(1) state_transition_done = True for s in isolated_states: if s['pgs_id'] != first_slave['id']: continue if s['active_role'] != '?' or s['mgmt_role'] != 'N': state_transition_done = False if state_transition_done : ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') # Unblock network self.assertTrue(util.iptables_drop('D', '127.0.0.100', first_slave['smr_mgmt_port']), 'delete a bloking role to iptables fail.') # Check cluster state ok = False for i in range(7): final_state = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) state_consistency = True for s in final_state: if s['pgs_id'] == 1: continue if is_pgs_normal(s) == False: state_consistency = False if state_consistency: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state consistency') ok = False for i in xrange(5): ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True) if ok: break else: time.sleep(1) self.assertTrue(ok, 'failed to check cluster state') # Check state self.assertNotEqual(initial_state, None, 'initial_state is None') self.assertNotEqual(final_state, None, 'final_state is None') initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id'])) final_state = sorted(final_state, key=lambda x: int(x['pgs_id'])) for i in range(len(final_state)): msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts']) util.log(msg) self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster)
def test_4_mgmt_is_isolated_with_red_failover(self): util.print_frame() util.iptables_print_list() cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0] util.log(util.json_to_str(cluster)) self.leader_cm = cluster['servers'][0] # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port) # Master must be the first pgs, cluster['servers'][0]. to_be_master = cluster['servers'][0] m = util.get_server_by_role_and_pg(cluster['servers'], 'master', to_be_master['pg_id']) master_id = -1 if m['id'] != to_be_master['id']: try_cnt = 0 while master_id != to_be_master['id'] and try_cnt < 20: master_id = util.role_change(cluster['servers'][0], cluster['cluster_name'], to_be_master['id']) try_cnt += 1 time.sleep(1) self.assertEquals(master_id, to_be_master['id'], 'change %d to a master fail' % to_be_master['id']) # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Set SMR option (slave_idle_timeout) util.log('\n\n\n ### Set SMR option ###') for s in cluster['servers']: t = telnet.Telnet('SMR%d' % s['id']) self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0, 'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port'])) cmd = 'confset slave_idle_timeout_msec 18000' util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd)) t.write('confset slave_idle_timeout_msec 18000\r\n') reply = t.read_until('\r\n').strip() util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply)) self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply) # Network isolation test for loop_cnt in range(3): # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt) for s in cluster['servers']: self.assertTrue(util.iptables_drop('A', '127.0.0.100', s['smr_mgmt_port']), 'add a bloking role to iptables fail.') for i in range(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in range(7): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) time.sleep(1) state_transition_done = True for s in isolated_states: if s['ip'] != '127.0.0.100': continue if s['active_role'] != '?' or s['mgmt_role'] != 'N': state_transition_done = False if state_transition_done : ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') pgs_list = util.get_pgs_info_list(mgmt_ip, mgmt_port, cluster) reds = filter(lambda x: x['color'] == 'RED', pgs_list) # Shutdown server = cluster['servers'][random.choice(reds)['pgs_id']] util.log( 'shutdown pgs%d while hanging.' % server['id'] ) ret = testbase.request_to_shutdown_smr( server ) self.assertEqual( ret, 0, 'failed to shutdown smr. id:%d' % server['id'] ) ret = testbase.request_to_shutdown_redis( server ) self.assertEqual( ret, 0, 'failed to shutdown redis. id:%d' % server['id'] ) # Check state F max_try = 20 expected = 'F' for i in range( 0, max_try): util.log('MGMT_IP:%s, MGMT_PORT:%d' % (mgmt_ip, mgmt_port)) state = util._get_smr_state( server['id'], cluster['cluster_name'], mgmt_ip, mgmt_port ) if expected == state: break; time.sleep( 1 ) self.assertEqual( expected , state, 'server%d - state:%s, expected:%s' % (server['id'], state, expected) ) util.log( 'succeeded : pgs%d state changed to F.' % server['id'] ) # Unblock network for s in cluster['servers']: self.assertTrue(util.iptables_drop('D', '127.0.0.100', s['smr_mgmt_port']), 'delete a bloking role to iptables fail.') # Check cluster state ok = False for i in range(10): final_state = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) state_consistency = True for s in final_state: if s['pgs_id'] == server['id']: continue if is_pgs_normal(s) == False: state_consistency = False if state_consistency: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state consistency') # Recovery util.log( 'restart pgs%d.' % server['id'] ) ret = testbase.request_to_start_smr( server ) self.assertEqual( ret, 0, 'failed to start smr. id:%d' % server['id'] ) ret = testbase.request_to_start_redis( server ) self.assertEqual( ret, 0, 'failed to start redis. id:%d' % server['id'] ) wait_count = 20 ret = testbase.wait_until_finished_to_set_up_role( server, wait_count ) self.assertEqual( ret, 0, 'failed to role change. smr_id:%d' % (server['id']) ) redis = redis_mgmt.Redis( server['id'] ) ret = redis.connect( server['ip'], server['redis_port'] ) self.assertEqual( ret, 0, 'failed to connect to redis' ) ok = False for i in xrange(5): ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True) if ok: break else: time.sleep(1) self.assertTrue(ok, 'failed to check cluster state') # Reset SMR option (slave_idle_timeout) t = telnet.Telnet('SMR%d' % server['id']) self.assertEqual(t.connect(server['ip'], server['smr_mgmt_port']), 0, 'Failed to connect to smr. ADDR=%s:%d' % (server['ip'], server['smr_mgmt_port'])) cmd = 'confset slave_idle_timeout_msec 18000' util.log('[%s:%d] >> %s' % (server['ip'], server['smr_mgmt_port'], cmd)) t.write('confset slave_idle_timeout_msec 18000\r\n') reply = t.read_until('\r\n').strip() util.log('[%s:%d] << %s' % (server['ip'], server['smr_mgmt_port'], reply)) self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply) # Check state self.assertNotEqual(initial_state, None, 'initial_state is None') self.assertNotEqual(final_state, None, 'final_state is None') initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id'])) final_state = sorted(final_state, key=lambda x: int(x['pgs_id'])) for i in range(len(final_state)): msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts']) util.log(msg) if initial_state[i]['pgs_id'] == 1: self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg) self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster)
def setUp(self): util.set_process_logfile_prefix( 'TestFreeClient_%s' % self._testMethodName ) ret = default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) if ret is not 0: default_cluster.finalize( self.cluster ) return 0
def test_6_repeat_isolation_and_no_opinion_linepay(self): util.print_frame() out = util.sudo('iptables -L') util.log('====================================================================') util.log('out : %s' % out) util.log('out.return_code : %d' % out.return_code) util.log('out.stderr : %s' % out.stderr) util.log('out.succeeded : %s' % out.succeeded) # Add forwarding role out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) cluster_name = 'no_opinion' cluster = filter(lambda x: x['cluster_name'] == cluster_name, config.clusters)[0] util.log(util.json_to_str(cluster)) self.leader_cm = cluster['servers'][0] # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster ret = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertEqual(0, ret, 'failed to TestMaintenance.initialize') # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Network isolation test loop_cnt = 0 while (loop_cnt < 20): loop_cnt += 1 # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt) out = util.sudo('iptables -A OUTPUT -d 127.0.0.102 -j DROP') self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out) for i in range(1): util.log('waiting... %d' % (i + 1)) time.sleep(0.1) out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP') self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out) for i in range(3): util.log('waiting... %d' % (i + 1)) time.sleep(1.2) out = util.sudo('iptables -A OUTPUT -d 127.0.0.101 -j DROP') self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out) for i in range(1): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % loop_cnt) out = util.sudo('iptables -D OUTPUT -d 127.0.0.102 -j DROP') self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out) for i in range(0): util.log('waiting... %d' % (i + 1)) time.sleep(1) out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP') self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out) for i in range(0): util.log('waiting... %d' % (i + 1)) time.sleep(1) out = util.sudo('iptables -D OUTPUT -d 127.0.0.101 -j DROP') self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out) for i in range(3): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Print state of cluster util.log('\n ### STATE OF CLUSTER ### ') cluster_state = False for i in range(10): cluster_state = util.check_cluster(cluster_name, mgmt_ip, mgmt_port, initial_state, check_quorum=True) if cluster_state == True: break else: time.sleep(1) self.assertTrue(cluster_state, 'failed to check cluster state') all_in_f = True for s in cluster['servers']: if checkLastState(mgmt_ip, s['cm_port'], cluster_name, 0, 'F') == False: all_in_f = False break self.assertFalse(all_in_f, 'PGS0`s last state remains in F') # Shutdown cluster ret = default_cluster.finalize( cluster ) self.assertEqual(ret, 0, 'failed to TestMaintenance.finalize') # Delete forwarding role out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.101 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.102 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
def setUp(self): util.set_process_logfile_prefix("TestGatewayMgmt_%s" % self._testMethodName) if default_cluster.initialize_starting_up_smr_before_redis(self.cluster) is not 0: util.log("failed to TestScaleout.initialize") return -1 return 0
def setUp(self): util.set_process_logfile_prefix( 'TestScaleout_%s' % self._testMethodName ) conf = {'smr_log_delete_delay':60} self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster, conf=conf ) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUp( self ): util.set_process_logfile_prefix( 'TestQuorumPolicy_%s' % self._testMethodName ) self.conf_checker = default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) self.assertIsNotNone(self.conf_checker, 'failed to initialize cluster')
def setUpClass( cls ): ret = default_cluster.initialize_starting_up_smr_before_redis( cls.cluster ) if ret is not 0: default_cluster.finalize( cls.cluster ) assert (ret == 0) return 0
def test_6_repeat_isolation_and_no_opinion_linepay(self): util.print_frame() util.iptables_print_list() # Add forwarding role self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.') self.assertTrue(util.iptables_redirect('A', '127.0.0.101', '127.0.0.1'), 'add a forwarding role to iptables fail.') self.assertTrue(util.iptables_redirect('A', '127.0.0.102', '127.0.0.1'), 'add a forwarding role to iptables fail.') cluster_name = 'no_opinion' cluster = filter(lambda x: x['cluster_name'] == cluster_name, config.clusters)[0] util.log(util.json_to_str(cluster)) self.leader_cm = cluster['servers'][0] # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Network isolation test loop_cnt = 0 while (loop_cnt < 20): loop_cnt += 1 # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt) self.assertTrue(util.iptables_drop('A', '127.0.0.102'), 'add a bloking role to iptables fail.') for i in range(1): util.log('waiting... %d' % (i + 1)) time.sleep(0.1) self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.') for i in range(3): util.log('waiting... %d' % (i + 1)) time.sleep(1.2) self.assertTrue(util.iptables_drop('A', '127.0.0.101'), 'add a bloking role to iptables fail.') for i in range(1): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % loop_cnt) self.assertTrue(util.iptables_drop('D', '127.0.0.102'), 'delete a bloking role to iptables fail.') for i in range(0): util.log('waiting... %d' % (i + 1)) time.sleep(1) self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.') for i in range(0): util.log('waiting... %d' % (i + 1)) time.sleep(1) self.assertTrue(util.iptables_drop('D', '127.0.0.101'), 'delete a bloking role to iptables fail.') for i in range(3): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Print state of cluster util.log('\n ### STATE OF CLUSTER ### ') cluster_state = False for i in range(10): cluster_state = util.check_cluster(cluster_name, mgmt_ip, mgmt_port, initial_state, check_quorum=True) if cluster_state == True: break else: time.sleep(1) self.assertTrue(cluster_state, 'failed to check cluster state') all_in_f = True for s in cluster['servers']: if checkLastState(mgmt_ip, s['cm_port'], cluster_name, 0, 'F') == False: all_in_f = False break self.assertFalse(all_in_f, 'PGS0`s last state remains in F') # Delete forwarding role self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail.') self.assertTrue(util.iptables_redirect('D', '127.0.0.101', '127.0.0.1'), 'delete a forwarding role to iptables fail.') self.assertTrue(util.iptables_redirect('D', '127.0.0.102', '127.0.0.1'), 'delete a forwarding role to iptables fail.') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster)
def test_7_dirty_network_fi(self): util.print_frame() clnts = [] try: util.iptables_print_list() # Add forwarding role self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.') cluster_name = 'network_isolation_cluster_1' cluster = filter(lambda x: x['cluster_name'] == cluster_name, config.clusters)[0] util.log(util.json_to_str(cluster)) self.leader_cm = cluster['servers'][0] # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster, conf={'cm_context':'applicationContext-fi.xml'}) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Start crc16 client for s in cluster['servers']: c = load_generator_crc16.Crc16Client(s['id'], s['ip'], s['redis_port'], 600, verbose=False) c.start() clnts.append(c) # Network isolation test cmfi = fi_confmaster.ConfmasterWfFi(['ra', 'me', 'yj', 'bj', 'mg'], ['lconn', 'slave', 'master', 'setquorum'], [True, False], 1) for fi in cmfi: # Block network util.log('\n\n\n ### BLOCK NETWORK, %s ### ' % str(fi)) ret = block_network(cluster, mgmt_ip, mgmt_port) self.assertTrue(ret, '[%s] failed to block network.' % str(fi)) for i in xrange(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in xrange(10): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) state_transition_done = True for s in isolated_states: if s['ip'] != '127.0.0.100': continue if s['active_role'] != '?' or s['mgmt_role'] != 'N': state_transition_done = False if state_transition_done: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') # Fault injection try: self.assertTrue(fi_confmaster.fi_add(fi, 1, mgmt_ip, mgmt_port), "Confmaster command fail. fi: %s" % str(fi)) except ValueError as e: self.fail("Confmaster command error. cmd: \"%s\", reply: \"%s\"" % (cmd, reply)) # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %s ### ' % str(fi)) ret = unblock_network(cluster, mgmt_ip, mgmt_port, None) self.assertTrue(ret, '[%s] failed to unblock network.' % str(fi)) for i in xrange(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in xrange(10): isolated_states = [] ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) if ok: break time.sleep(1) self.assertTrue(ok, '[%s] Fail. unstable cluster.' % str(fi)) check_cluster = False # 'bj', 'slave' if fi[0] == 'bj' and fi[1] == 'slave': m, s1, s2 = util.get_mss(cluster) ret = util.role_lconn(s1) self.assertEqual("+OK\r\n", ret, '[%s] role lconn fail.' % str(fi)) check_cluster = True # 'me', 'lconn' elif fi[0] == 'me': m, s1, s2 = util.get_mss(cluster) ret = util.role_lconn(m) self.assertEqual("+OK\r\n", ret, '[%s] role lconn fail.' % str(fi)) check_cluster = True # 'setquorum' elif fi[1] == 'setquorum': m, s1, s2 = util.get_mss(cluster) ret = util.cmd_to_smr_addr(s1['ip'], s1['smr_mgmt_port'], 'fi delay sleep 1 8000\r\n', timeout=20) self.assertEqual("+OK\r\n", ret, '[%s] "fi delay sleep 1 8000" fail. ret: "%s"' % (str(fi), ret)) check_cluster = True if check_cluster: # Check cluster state ok = False for i in xrange(20): isolated_states = [] ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) if ok: break time.sleep(1) self.assertTrue(ok, '[%s] Fail. unstable cluster.' % str(fi)) # Check fault injection ok = False for i in xrange(10): count = fi_confmaster.fi_count(fi, mgmt_ip, mgmt_port) if count == 0: ok = True break time.sleep(0.5) self.assertTrue(ok, "[%s] fail. failt injection had not been triggered." % str(fi)) for c in clnts: self.assertTrue(c.is_consistency(), '[%s] data consistency error!' % str(fi)) for c in clnts: self.assertTrue(c.is_consistency(), '[%s] data consistency error!' % str(fi)) # Go back to initial configuration cmfi.init() for fi in cmfi: try: self.assertTrue(fi_confmaster.fi_add(fi, 0, mgmt_ip, mgmt_port), "Confmaster command fail. fi: %s" % str(fi)) except ValueError as e: self.fail("Confmaster command error. cmd: \"%s\", reply: \"%s\"" % (cmd, reply)) # Wait until workflows done ret = util.await(60, True)( lambda cinfo: cinfo['wf'] == 0, lambda : util.cluster_info(mgmt_ip, mgmt_port, cluster['cluster_name'])) self.assertTrue(ret, 'There are still some workflows.') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster) finally: for c in clnts: c.quit() for c in clnts: c.join() # Delete forwarding role self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
def test_1_mgmt_is_isolated(self): util.print_frame() util.iptables_print_list() cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0] util.log(util.json_to_str(cluster)) # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Set SMR option (slave_idle_timeout) util.log('\n\n\n ### Set SMR option ###') for s in cluster['servers']: t = telnet.Telnet('SMR%d' % s['id']) self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0, 'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port'])) cmd = 'confset slave_idle_timeout_msec 18000' util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd)) t.write('confset slave_idle_timeout_msec 18000\r\n') reply = t.read_until('\r\n').strip() util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply)) self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply) # Network isolation test for cnt in range(5): # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt) for s in cluster['servers']: """Loopback Address Range (Reference : RFC3330) 127.0.0.0/8 - This block is assigned for use as the Internet host loopback address. A datagram sent by a higher level protocol to an address anywhere within this block should loop back inside the host. This is ordinarily implemented using only 127.0.0.1/32 for loopback, but no addresses within this block should ever appear on any network anywhere [RFC1700, page 5]. """ self.assertTrue(util.iptables_drop('A', '127.0.0.100', s['smr_mgmt_port']), 'add a bloking role to iptables fail.') for i in range(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in range(7): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) time.sleep(1) state_transition_done = True for s in isolated_states: if s['ip'] != '127.0.0.100': continue if s['active_role'] != '?' or s['mgmt_role'] != 'N': state_transition_done = False if state_transition_done : ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt) for s in cluster['servers']: self.assertTrue(util.iptables_drop('D', '127.0.0.100', s['smr_mgmt_port']), 'delete a bloking role to iptables fail.') # Check cluster state ok = False for i in range(7): final_state = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) all_green = True for s in final_state: if is_pgs_normal(s) == False: all_green = False if all_green: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state consistency') # Check state self.assertNotEqual(initial_state, None, 'initial_state is None') self.assertNotEqual(final_state, None, 'final_state is None') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster)
def test_2_some_pgs_is_isolated(self): util.print_frame() util.iptables_print_list() # Add forwarding role (127.0.0.100 -> 127.0.0.1) self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.') cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_2', config.clusters)[0] util.log(util.json_to_str(cluster)) # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') # Place master on virtual ip address in order to cause master election. pg_id = 0 m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id) s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id) if m.has_key('ip') == True and m.has_key('real_ip') == False: ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id']) self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id']) # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Network isolation test for cnt in range(3): # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt) self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.') for i in range(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in range(7): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) time.sleep(1) state_transition_done = True for s in isolated_states: if s['pgs_id'] == 1: continue if s['active_role'] != '?' or s['mgmt_role'] != 'N': state_transition_done = False if state_transition_done : ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt) self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.') # Check cluster state ok = False for i in range(7): final_state = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) state_consistency = True for s in final_state: if s['pgs_id'] == 1: continue if is_pgs_normal(s) == False: state_consistency = False if state_consistency: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state consistency') # Check state self.assertNotEqual(initial_state, None, 'initial_state is None') self.assertNotEqual(final_state, None, 'final_state is None') initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id'])) final_state = sorted(final_state, key=lambda x: int(x['pgs_id'])) for i in range(len(final_state)): msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts']) util.log(msg) self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg) # Delete forwarding role (127.0.0.100 -> 127.0.0.1) self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster)
def setUp(self): util.set_process_logfile_prefix( 'TestMaintenance_%s' % self._testMethodName ) if default_cluster.initialize_starting_up_smr_before_redis( self.cluster ) is not 0: util.log('failed to TestMaintenance.initialize') return -1 return 0
def test_5_mgmt_is_isolated_with_master_failover(self): util.print_frame() out = util.sudo('iptables -L') util.log('====================================================================') util.log('out : %s' % out) util.log('out.return_code : %d' % out.return_code) util.log('out.stderr : %s' % out.stderr) util.log('out.succeeded : %s' % out.succeeded) # Add forwarding role (127.0.0.100 -> 127.0.0.1) out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out) cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0] util.log(util.json_to_str(cluster)) self.leader_cm = cluster['servers'][0] # MGMT mgmt_ip = cluster['servers'][0]['real_ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster ret = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertEqual(0, ret, 'failed to TestMaintenance.initialize') # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Network isolation test for loop_cnt in range(3): master, slave1, slave2 = util.get_mss(cluster) self.assertNotEquals(master, None, 'there is no master') self.assertNotEquals(slave1, None, 'there is no slave1') self.assertNotEquals(slave2, None, 'there is no slave2') # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt) out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP') self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out) for i in range(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in range(10): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) time.sleep(1) state_transition_done = True for s in isolated_states: if s['ip'] != '127.0.0.100': continue if s['active_role'] != '?' or s['mgmt_role'] != 'N': state_transition_done = False if state_transition_done : ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') # Shutdown master util.log( 'shutdown pgs%d while hanging.' % master['id'] ) ret = testbase.request_to_shutdown_smr( master ) self.assertEqual( ret, 0, 'failed to shutdown smr. id:%d' % master['id'] ) ret = testbase.request_to_shutdown_redis( master ) self.assertEqual( ret, 0, 'failed to shutdown redis. id:%d' % master['id'] ) # Check state F max_try = 20 expected = 'F' for i in range( 0, max_try): util.log('MGMT_IP:%s, MGMT_PORT:%d' % (mgmt_ip, mgmt_port)) state = util._get_smr_state( master['id'], cluster['cluster_name'], mgmt_ip, mgmt_port ) if expected == state: break; time.sleep( 1 ) self.assertEqual( expected , state, 'master%d - state:%s, expected:%s' % (master['id'], state, expected) ) util.log( 'succeeded : pgs%d state changed to F.' % master['id'] ) # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % loop_cnt) out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP') self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out) # Check cluster state ok = False for i in range(7): final_state = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) state_consistency = True for s in final_state: if s['pgs_id'] == master['id']: continue if s['active_role'] != s['mgmt_role']: state_consistency = False if state_consistency: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state consistency') # Recovery util.log( 'restart pgs%d.' % master['id'] ) ret = testbase.request_to_start_smr( master ) self.assertEqual( ret, 0, 'failed to start smr. id:%d' % master['id'] ) ret = testbase.request_to_start_redis( master ) self.assertEqual( ret, 0, 'failed to start redis. id:%d' % master['id'] ) wait_count = 20 ret = testbase.wait_until_finished_to_set_up_role( master, wait_count ) self.assertEqual( ret, 0, 'failed to role change. smr_id:%d' % (master['id']) ) redis = redis_mgmt.Redis( master['id'] ) ret = redis.connect( master['ip'], master['redis_port'] ) self.assertEqual( ret, 0, 'failed to connect to redis' ) ok = False for i in xrange(5): ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True) if ok: break else: time.sleep(1) self.assertTrue(ok, 'failed to check cluster state') # Check state self.assertNotEqual(initial_state, None, 'initial_state is None') self.assertNotEqual(final_state, None, 'final_state is None') initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id'])) final_state = sorted(final_state, key=lambda x: int(x['pgs_id'])) for i in range(0, 3): msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts']) util.log(msg) self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg) for i in range(3, 6): msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts']) util.log(msg) self.assertEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg) self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state') # Shutdown cluster ret = default_cluster.finalize( cluster ) self.assertEqual(ret, 0, 'failed to TestMaintenance.finalize') # Delete forwarding role (127.0.0.100 -> 127.0.0.1) out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out) out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1') self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
def test_3_some_pgs_is_isolated_2copy(self): util.print_frame() out = util.sudo('iptables -L') util.log('====================================================================') util.log('out : %s' % out) util.log('out.return_code : %d' % out.return_code) util.log('out.stderr : %s' % out.stderr) util.log('out.succeeded : %s' % out.succeeded) # Add forwarding role (127.0.0.100 -> 127.0.0.1) self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.') cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1_2copy', config.clusters)[0] util.log(util.json_to_str(cluster)) # MGMT mgmt_ip = cluster['servers'][0]['ip'] mgmt_port = cluster['servers'][0]['cm_port'] # Create cluster conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster ) self.assertIsNotNone(conf_checker, 'failed to initialize cluster') # Place master on real ip address for pg_id in [0, 1]: m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id) s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id) if m.has_key('ip') and m.has_key('real_ip'): if m['ip'] != m['real_ip']: ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id']) self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id']) # Print initial state of cluster util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ') initial_state = [] self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state') # Network isolation test for cnt in range(3): # Block network util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt) self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.') for i in range(4): util.log('waiting... %d' % (i + 1)) time.sleep(1) # Check cluster state ok = False for i in range(7): isolated_states = [] util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True) time.sleep(1) state_transition_done = True for s in isolated_states: if s['pgs_id'] == 0 or s['pgs_id'] == 1: continue if s['active_role'] != 'M' or s['mgmt_role'] != 'M': state_transition_done = False if s['quorum'] != 0: state_transition_done = False if state_transition_done: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state transition') # Unblock network util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt) self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.') # Check cluster state ok = False for i in range(7): final_state = [] if util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) == False: time.sleep(1) continue state_consistency = True for s in final_state: if s['pgs_id'] == 1: continue if is_pgs_normal(s) == False: state_consistency = False if state_consistency: ok = True break time.sleep(1) self.assertTrue(ok, 'Fail, state consistency') # Check state self.assertNotEqual(initial_state, None, 'initial_state is None') self.assertNotEqual(final_state, None, 'final_state is None') # Delete forwarding role (127.0.0.100 -> 127.0.0.1) self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail.') self.assertTrue(conf_checker.final_check()) # Shutdown cluster default_cluster.finalize(cluster)