def test_4_role_change_with_failover(self):
util.print_frame()
loop_cnt = 0
while loop_cnt < 5:
util.log('')
util.log('Loop:%d' % loop_cnt)
util.log("States (before role change)")
util.log_server_state(self.cluster)
target = random.choice(self.cluster['servers'])
# bgsave
ret = util.bgsave(target)
self.assertTrue(ret, 'failed to bgsave. pgs:%d' % target['id'])
# shutdown
util.log('shutdown pgs%d(%s:%d)' %
(target['id'], target['ip'], target['smr_base_port']))
ret = testbase.request_to_shutdown_smr(target)
self.assertEqual(ret, 0, 'failed to shutdown smr')
ret = testbase.request_to_shutdown_redis(target)
self.assertEquals(ret, 0, 'failed to shutdown redis')
r = ''
expected = 'N'
for fc_cnt in xrange(20):
r = util.get_smr_role_of_cm(target, self.leader_cm)
if r == expected:
break
time.sleep(0.5)
self.assertEquals(r, expected, 'failure detection error.')
running_servers = []
for s in self.cluster['servers']:
if s != target:
running_servers.append(s)
# Get old timestamp
old_timestamps = {}
for s in running_servers:
ts = util.get_timestamp_of_pgs(s)
old_timestamps[s['id']] = ts
# Start load generator
self.load_gen_list = {}
util.log('start load generator')
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
m, s1, s2 = util.get_mss(self.cluster)
self.assertNotEqual(m, None, 'master is None.')
self.assertNotEqual(s1, None, 'slave1 is None.')
# Role change
master_id = util.role_change(self.leader_cm,
self.cluster['cluster_name'],
s1['id'])
self.assertNotEqual(master_id, -1, 'role_change failed')
util.log("States (after role change)")
util.log_server_state(self.cluster)
# Check - get new timestamp
new_timestamps = {}
for s in running_servers:
ts = util.get_timestamp_of_pgs(s)
new_timestamps[s['id']] = ts
# Check - compare old timestamps and new timestamps
for s in running_servers:
old_ts = old_timestamps[s['id']]
new_ts = new_timestamps[s['id']]
self.assertNotEqual(
old_ts, new_ts,
'Timestamp of a running server has not changed. %d->%d' %
(old_ts, new_ts))
# Check quorum
m = self.cluster['servers'][master_id]
expected = 1
ok = self.__check_quorum(m, expected)
self.assertTrue(
ok, 'unexpected quorum(after role change). expected:%s' %
(expected))
# recovery
util.log('recovery pgs%d(%s:%d)' %
(target['id'], target['ip'], target['smr_base_port']))
ret = testbase.request_to_start_smr(target)
self.assertEqual(ret, 0, 'failed to start smr')
util.log('start smr-replicator done')
ret = testbase.request_to_start_redis(target, 60)
self.assertEqual(ret, 0, 'failed to start redis')
util.log('start redis-arc done')
ret = testbase.wait_until_finished_to_set_up_role(target,
max_try=300)
self.assertEquals(
ret, 0, 'failed to role change. smr_id:%d' % (target['id']))
util.log("States (after recovery)")
util.log_server_state(self.cluster)
# Check cluster state
normal_state = False
for i in xrange(20):
normal_state = util.check_cluster(self.cluster['cluster_name'],
self.leader_cm['ip'],
self.leader_cm['cm_port'],
check_quorum=True)
if normal_state:
break
time.sleep(0.5)
self.assertTrue(normal_state, "Unstable cluster state")
# Check quorum
expected = 2
ok = self.__check_quorum(m, expected)
self.assertTrue(
ok,
'unexpected quorum(after recovery). expected:%s' % (expected))
# Cheeck Consistency
util.log('stop load generator')
for i in range(self.max_load_generator):
self.load_gen_list[i].quit()
for i in range(self.max_load_generator):
self.load_gen_list[i].join()
self.assertTrue(self.load_gen_list[i].isConsistent(),
'Inconsistent after migration')
self.load_gen_list.pop(i, None)
loop_cnt += 1
return 0
def test_1_role_change(self):
util.print_frame()
self.load_gen_list = {}
# Start load generator
util.log("Start load_generator")
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
# Loop (smr: 3 copy)
for i in range(30):
target_server = util.get_server_by_role(self.cluster['servers'],
'slave')
self.assertNotEquals(target_server, None, 'Get slave fail.')
target = target_server['id']
print ''
util.log("(3 copy) Loop:%d, target pgs:%d" % (i, target))
# Get old timestamp
util.log_server_state(self.cluster)
old_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs(s)
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm,
self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target + 1) % 3
util.log('Change role success.')
# Wait until role change finished
for s in self.cluster['servers']:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
try:
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
except:
pass
time.sleep(0.2)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state(self.cluster)
new_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs(s)
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(3):
self.assertNotEqual(
old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' %
(old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(),
'Data inconsistency after role_change')
# Loop (smr: 2 copy)
self.__del_server(self.cluster['servers'][0])
servers = [self.cluster['servers'][1], self.cluster['servers'][2]]
normal_state = False
for i in xrange(20):
normal_state = util.check_cluster(self.cluster['cluster_name'],
self.leader_cm['ip'],
self.leader_cm['cm_port'],
check_quorum=True)
if normal_state:
break
time.sleep(0.5)
self.assertTrue(normal_state, "Unstable cluster state")
for i in range(30):
print ''
util.log("(2 copy) Loop:%d, target pgs:%d" % (i, target))
s = util.get_server_by_role(servers, 'slave')
target = s['id']
# Get old timestamp
util.log_server_state(self.cluster)
old_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs(s)
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm,
self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target) % 2 + 1
util.log('Change role success.')
# Wait until role change finished
for s in servers:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
time.sleep(0.1)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state(self.cluster)
new_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs(s)
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(2):
self.assertNotEqual(
old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' %
(old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(),
'Data inconsistency after role_change')
# Go back to initial configuration
self.assertTrue(
util.install_pgs(self.cluster,
self.cluster['servers'][0],
self.leader_cm,
rm_ckpt=False), 'failed to recover pgs.')
def role_change_with_hanging_pgs(self, hanging_servers, running_servers,
target_id, master):
util.log('hanging_servers:%s' % hanging_servers)
util.log('running_servers:%s' % running_servers)
util.log('target_id:%s' % target_id)
# Initial data
util.put_some_data(self.cluster, 3, 10)
util.log("States (before role change)")
util.log_server_state(self.cluster)
# Get old timestamp
old_timestamps = {}
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs(s)
old_timestamps[s['id']] = ts
# hang
for s in hanging_servers:
smr = smr_mgmt.SMR(s['id'])
ret = smr.connect(s['ip'], s['smr_mgmt_port'])
self.assertEqual(
ret, 0, 'failed to connect to master. %s:%d' %
(s['ip'], s['smr_mgmt_port']))
util.log("PGS '%d' hang" % s['id'])
smr.write('fi delay sleep 1 13000\r\n')
reply = smr.read_until('\r\n', 1)
if reply != None and reply.find('-ERR not supported') != -1:
self.assertEqual(
0, 1, 'make sure that smr has compiled with gcov option.')
smr.disconnect()
# Role change
master_id = util.role_change(self.leader_cm,
self.cluster['cluster_name'], target_id)
self.assertEqual(master_id, -1,
'We expected that role_change failed, but success')
# Check rollback - check quorum
if master not in hanging_servers:
expected = 2
ok = self.__check_quorum(master, expected)
self.assertTrue(ok,
'rollback quorum fail. expected:%s' % (expected))
# Check rollback - get new timestamp
new_timestamps_in_hang = {}
for s in running_servers:
ts = util.get_timestamp_of_pgs(s)
new_timestamps_in_hang[s['id']] = ts
# Check rollback - compare old timestamps and new timestamps
for s in running_servers:
old_ts = old_timestamps[s['id']]
new_ts = new_timestamps_in_hang[s['id']]
self.assertEqual(
old_ts, new_ts,
'Timestamp of a running server has changed. %d->%d' %
(old_ts, new_ts))
time.sleep(16)
util.log("States (after role change)")
util.log_server_state(self.cluster)
self.load_gen_list = {}
# Start load generator
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
# Check quorum
if master in hanging_servers:
m, s1, s2 = util.get_mss(self.cluster)
self.assertNotEqual(m, None, 'master is None.')
self.assertNotEqual(s1, None, 'slave1 is None.')
self.assertNotEqual(s2, None, 'slave2 is None.')
expected = 2
ok = self.__check_quorum(m, expected)
self.assertTrue(ok,
'rollback quorum fail. expected:%s' % (expected))
# Check cluster state
normal_state = False
for i in xrange(20):
normal_state = util.check_cluster(self.cluster['cluster_name'],
self.leader_cm['ip'],
self.leader_cm['cm_port'],
check_quorum=True)
if normal_state:
break
time.sleep(0.5)
self.assertTrue(normal_state, "Unstable cluster state")
# Cheeck Consistency
for i in range(self.max_load_generator):
self.load_gen_list[i].quit()
for i in range(self.max_load_generator):
self.load_gen_list[i].join()
self.assertTrue(self.load_gen_list[i].isConsistent(),
'Inconsistent after migration')
self.load_gen_list.pop(i, None)
def test_3_some_pgs_is_isolated_2copy(self):
util.print_frame()
out = util.sudo('iptables -L')
util.log('====================================================================')
util.log('out : %s' % out)
util.log('out.return_code : %d' % out.return_code)
util.log('out.stderr : %s' % out.stderr)
util.log('out.succeeded : %s' % out.succeeded)
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1_2copy', config.clusters)[0]
util.log(util.json_to_str(cluster))
# MGMT
mgmt_ip = cluster['servers'][0]['ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Place master on real ip address
for pg_id in [0, 1]:
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id)
s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id)
if m.has_key('ip') and m.has_key('real_ip'):
if m['ip'] != m['real_ip']:
ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id'])
self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
for cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['pgs_id'] == 0 or s['pgs_id'] == 1:
continue
if s['active_role'] != 'M' or s['mgmt_role'] != 'M':
state_transition_done = False
if s['quorum'] != 0:
state_transition_done = False
if state_transition_done:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(7):
final_state = []
if util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True) == False:
time.sleep(1)
continue
state_consistency = True
for s in final_state:
if s['pgs_id'] == 1:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
# Delete forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail.')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def test_4_mgmt_is_isolated_with_red_failover(self):
util.print_frame()
util.iptables_print_list()
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_1', config.clusters)[0]
util.log(util.json_to_str(cluster))
self.leader_cm = cluster['servers'][0]
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port)
# Master must be the first pgs, cluster['servers'][0].
to_be_master = cluster['servers'][0]
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', to_be_master['pg_id'])
master_id = -1
if m['id'] != to_be_master['id']:
try_cnt = 0
while master_id != to_be_master['id'] and try_cnt < 20:
master_id = util.role_change(cluster['servers'][0], cluster['cluster_name'], to_be_master['id'])
try_cnt += 1
time.sleep(1)
self.assertEquals(master_id, to_be_master['id'], 'change %d to a master fail' % to_be_master['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Set SMR option (slave_idle_timeout)
util.log('\n\n\n ### Set SMR option ###')
for s in cluster['servers']:
t = telnet.Telnet('SMR%d' % s['id'])
self.assertEqual(t.connect(s['ip'], s['smr_mgmt_port']), 0,
'Failed to connect to smr. ADDR=%s:%d' % (s['ip'], s['smr_mgmt_port']))
cmd = 'confset slave_idle_timeout_msec 18000'
util.log('[%s:%d] >> %s' % (s['ip'], s['smr_mgmt_port'], cmd))
t.write('confset slave_idle_timeout_msec 18000\r\n')
reply = t.read_until('\r\n').strip()
util.log('[%s:%d] << %s' % (s['ip'], s['smr_mgmt_port'], reply))
self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply)
# Network isolation test
for loop_cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % loop_cnt)
for s in cluster['servers']:
self.assertTrue(util.iptables_drop('A', '127.0.0.100', s['smr_mgmt_port']), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['ip'] != '127.0.0.100':
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
pgs_list = util.get_pgs_info_list(mgmt_ip, mgmt_port, cluster)
reds = filter(lambda x: x['color'] == 'RED', pgs_list)
# Shutdown
server = cluster['servers'][random.choice(reds)['pgs_id']]
util.log( 'shutdown pgs%d while hanging.' % server['id'] )
ret = testbase.request_to_shutdown_smr( server )
self.assertEqual( ret, 0, 'failed to shutdown smr. id:%d' % server['id'] )
ret = testbase.request_to_shutdown_redis( server )
self.assertEqual( ret, 0, 'failed to shutdown redis. id:%d' % server['id'] )
# Check state F
max_try = 20
expected = 'F'
for i in range( 0, max_try):
util.log('MGMT_IP:%s, MGMT_PORT:%d' % (mgmt_ip, mgmt_port))
state = util._get_smr_state( server['id'], cluster['cluster_name'], mgmt_ip, mgmt_port )
if expected == state:
break;
time.sleep( 1 )
self.assertEqual( expected , state,
'server%d - state:%s, expected:%s' % (server['id'], state, expected) )
util.log( 'succeeded : pgs%d state changed to F.' % server['id'] )
# Unblock network
for s in cluster['servers']:
self.assertTrue(util.iptables_drop('D', '127.0.0.100', s['smr_mgmt_port']), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(10):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == server['id']:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Recovery
util.log( 'restart pgs%d.' % server['id'] )
ret = testbase.request_to_start_smr( server )
self.assertEqual( ret, 0, 'failed to start smr. id:%d' % server['id'] )
ret = testbase.request_to_start_redis( server )
self.assertEqual( ret, 0, 'failed to start redis. id:%d' % server['id'] )
wait_count = 20
ret = testbase.wait_until_finished_to_set_up_role( server, wait_count )
self.assertEqual( ret, 0, 'failed to role change. smr_id:%d' % (server['id']) )
redis = redis_mgmt.Redis( server['id'] )
ret = redis.connect( server['ip'], server['redis_port'] )
self.assertEqual( ret, 0, 'failed to connect to redis' )
ok = False
for i in xrange(5):
ok = util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True)
if ok:
break
else:
time.sleep(1)
self.assertTrue(ok, 'failed to check cluster state')
# Reset SMR option (slave_idle_timeout)
t = telnet.Telnet('SMR%d' % server['id'])
self.assertEqual(t.connect(server['ip'], server['smr_mgmt_port']), 0,
'Failed to connect to smr. ADDR=%s:%d' % (server['ip'], server['smr_mgmt_port']))
cmd = 'confset slave_idle_timeout_msec 18000'
util.log('[%s:%d] >> %s' % (server['ip'], server['smr_mgmt_port'], cmd))
t.write('confset slave_idle_timeout_msec 18000\r\n')
reply = t.read_until('\r\n').strip()
util.log('[%s:%d] << %s' % (server['ip'], server['smr_mgmt_port'], reply))
self.assertEqual(reply, '+OK', 'Failed to set slave_idle_timeout, REPLY=%s' % reply)
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(len(final_state)):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
if initial_state[i]['pgs_id'] == 1:
self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, check_quorum=True), 'failed to check cluster state')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def test_2_some_pgs_is_isolated(self):
util.print_frame()
util.iptables_print_list()
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_2', config.clusters)[0]
util.log(util.json_to_str(cluster))
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
conf_checker = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertIsNotNone(conf_checker, 'failed to initialize cluster')
# Place master on virtual ip address in order to cause master election.
pg_id = 0
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id)
s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id)
if m.has_key('ip') == True and m.has_key('real_ip') == False:
ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id'])
self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
for cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['pgs_id'] == 1:
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt)
self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')
# Check cluster state
ok = False
for i in range(7):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == 1:
continue
if is_pgs_normal(s) == False:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(len(final_state)):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
# Delete forwarding role (127.0.0.100 -> 127.0.0.1)
self.assertTrue(util.iptables_redirect('D', '127.0.0.100', '127.0.0.1'), 'delete a forwarding role to iptables fail')
self.assertTrue(conf_checker.final_check())
# Shutdown cluster
default_cluster.finalize(cluster)
def test_4_role_change_with_failover(self):
util.print_frame()
loop_cnt = 0
while loop_cnt < 5:
util.log('')
util.log('Loop:%d' % loop_cnt)
util.log("States (before role change)")
util.log_server_state(self.cluster)
target = random.choice(self.cluster['servers'])
# bgsave
ret = util.bgsave(target)
self.assertTrue(ret, 'failed to bgsave. pgs:%d' % target['id'])
# shutdown
util.log('shutdown pgs%d(%s:%d)' % (target['id'], target['ip'], target['smr_base_port']))
ret = testbase.request_to_shutdown_smr( target )
self.assertEqual( ret, 0, 'failed to shutdown smr' )
ret = testbase.request_to_shutdown_redis( target )
self.assertEquals( ret, 0, 'failed to shutdown redis' )
running_servers = []
for s in self.cluster['servers']:
if s != target:
running_servers.append(s)
# Get old timestamp
old_timestamps = {}
for s in running_servers:
ts = util.get_timestamp_of_pgs(s)
old_timestamps[s['id']] = ts
# Start load generator
self.load_gen_list = {}
util.log('start load generator')
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
m, s1, s2 = util.get_mss(self.cluster)
self.assertNotEqual(m, None, 'master is None.')
self.assertNotEqual(s1, None, 'slave1 is None.')
# Role change
master_id = util.role_change(self.leader_cm, self.cluster['cluster_name'], s1['id'])
self.assertNotEqual(master_id, -1, 'role_change failed')
util.log("States (after role change)")
util.log_server_state(self.cluster)
# Check - get new timestamp
new_timestamps= {}
for s in running_servers:
ts = util.get_timestamp_of_pgs( s )
new_timestamps[s['id']] = ts
# Check - compare old timestamps and new timestamps
for s in running_servers:
old_ts = old_timestamps[s['id']]
new_ts = new_timestamps[s['id']]
self.assertNotEqual(old_ts, new_ts, 'Timestamp of a running server has not changed. %d->%d' % (old_ts, new_ts))
# Check quorum
m = self.cluster['servers'][master_id]
expected = 1
ok = self.__check_quorum(m, expected)
self.assertTrue(ok, 'unexpected quorum(after role change). expected:%s' % (expected))
# recovery
util.log('recovery pgs%d(%s:%d)' % (target['id'], target['ip'], target['smr_base_port']))
ret = testbase.request_to_start_smr( target )
self.assertEqual( ret, 0, 'failed to start smr' )
util.log('start smr-replicator done')
ret = testbase.request_to_start_redis( target, 60 )
self.assertEqual( ret, 0, 'failed to start redis' )
util.log('start redis-arc done')
ret = testbase.wait_until_finished_to_set_up_role( target, max_try=300)
self.assertEquals( ret, 0, 'failed to role change. smr_id:%d' % (target['id']) )
util.log("States (after recovery)")
util.log_server_state(self.cluster)
# Check quorum
expected = 1
ok = self.__check_quorum(m, expected)
self.assertTrue(ok, 'unexpected quorum(after recovery). expected:%s' % (expected))
# Cheeck Consistency
util.log('stop load generator')
for i in range(self.max_load_generator):
self.load_gen_list[i].quit()
for i in range(self.max_load_generator):
self.load_gen_list[i].join()
self.assertTrue(self.load_gen_list[i].isConsistent(), 'Inconsistent after migration')
self.load_gen_list.pop(i, None)
loop_cnt += 1
return 0
def role_change_with_hanging_pgs(self, hanging_servers, running_servers, target_id, master):
util.log('hanging_servers:%s' % hanging_servers)
util.log('running_servers:%s' % running_servers)
util.log('target_id:%s' % target_id)
# Initial data
util.put_some_data(self.cluster, 3, 10)
util.log("States (before role change)")
util.log_server_state(self.cluster)
# Get old timestamp
old_timestamps = {}
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs(s)
old_timestamps[s['id']] = ts
# hang
for s in hanging_servers:
smr = smr_mgmt.SMR(s['id'])
ret = smr.connect(s['ip'], s['smr_mgmt_port'])
self.assertEqual(ret, 0, 'failed to connect to master. %s:%d' % (s['ip'], s['smr_mgmt_port']))
util.log("PGS '%d' hang" % s['id'])
smr.write('fi delay sleep 1 13000\r\n')
reply = smr.read_until('\r\n', 1)
if reply != None and reply.find('-ERR not supported') != -1:
self.assertEqual(0, 1, 'make sure that smr has compiled with gcov option.')
smr.disconnect()
# Role change
master_id = util.role_change(self.leader_cm, self.cluster['cluster_name'], target_id)
self.assertEqual(master_id, -1, 'We expected that role_change failed, but success')
# Check rollback - check quorum
if master not in hanging_servers:
expected = 1
ok = self.__check_quorum(master, expected)
self.assertTrue(ok, 'rollback quorum fail. expected:%s' % (expected))
# Check rollback - get new timestamp
new_timestamps_in_hang = {}
for s in running_servers:
ts = util.get_timestamp_of_pgs( s )
new_timestamps_in_hang[s['id']] = ts
# Check rollback - compare old timestamps and new timestamps
for s in running_servers:
old_ts = old_timestamps[s['id']]
new_ts = new_timestamps_in_hang[s['id']]
self.assertEqual(old_ts, new_ts, 'Timestamp of a running server has changed. %d->%d' % (old_ts, new_ts))
time.sleep(16)
util.log("States (after role change)")
util.log_server_state( self.cluster )
self.load_gen_list = {}
# Start load generator
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
# Check quorum
if master in hanging_servers:
m, s1, s2 = util.get_mss(self.cluster)
self.assertNotEqual(m, None, 'master is None.')
self.assertNotEqual(s1, None, 'slave1 is None.')
self.assertNotEqual(s2, None, 'slave2 is None.')
expected = 1
ok = self.__check_quorum(m, expected)
self.assertTrue(ok, 'rollback quorum fail. expected:%s' % (expected))
# Get new timestamp
new_timestamps = {}
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs( s )
new_timestamps[s['id']] = ts
# Compare old timestamps and new timestamps
for s in self.cluster['servers']:
old_ts = old_timestamps[s['id']]
new_ts = new_timestamps[s['id']]
if master in hanging_servers and len(running_servers) != 0:
self.assertNotEqual(old_ts, new_ts, 'Timestamp of a hanging server has not changed. %d->%d' % (old_ts, new_ts))
else:
self.assertEqual(old_ts, new_ts, 'Timestamp of a running server has changed. %d->%d' % (old_ts, new_ts))
# Cheeck Consistency
for i in range(self.max_load_generator):
self.load_gen_list[i].quit()
for i in range(self.max_load_generator):
self.load_gen_list[i].join()
self.assertTrue(self.load_gen_list[i].isConsistent(), 'Inconsistent after migration')
self.load_gen_list.pop(i, None)
def test_1_role_change(self):
util.print_frame()
self.load_gen_list = {}
# Start load generator
util.log("Start load_generator")
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
# Loop (smr: 3 copy)
target_server = util.get_server_by_role(self.cluster['servers'], 'slave')
self.assertNotEquals(target_server, None, 'Get slave fail.')
target = target_server['id']
for i in range(30):
print ''
util.log("(3 copy) Loop:%d, target pgs:%d" % (i, target))
# Get old timestamp
util.log_server_state( self.cluster )
old_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs( s )
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm, self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target + 1) % 3
util.log('Change role success.')
# Wait until role change finished
for s in self.cluster['servers']:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
time.sleep(0.1)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state( self.cluster )
new_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs( s )
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(3):
self.assertNotEqual(old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' % (old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(), 'Data inconsistency after role_change')
# Loop (smr: 2 copy)
self.__del_server(self.cluster['servers'][0])
servers = [self.cluster['servers'][1], self.cluster['servers'][2]]
s = util.get_server_by_role(servers, 'slave')
target = s['id']
for i in range(30):
print ''
util.log("(2 copy) Loop:%d, target pgs:%d" % (i, target))
# Get old timestamp
util.log_server_state( self.cluster )
old_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs( s )
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm, self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target) % 2 + 1
util.log('Change role success.')
# Wait until role change finished
for s in servers:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
time.sleep(0.1)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state( self.cluster )
new_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs( s )
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(2):
self.assertNotEqual(old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' % (old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(), 'Data inconsistency after role_change')
def test_2_some_pgs_is_isolated(self):
util.print_frame()
out = util.sudo('iptables -L')
util.log('====================================================================')
util.log('out : %s' % out)
util.log('out.return_code : %d' % out.return_code)
util.log('out.stderr : %s' % out.stderr)
util.log('out.succeeded : %s' % out.succeeded)
# Add forwarding role (127.0.0.100 -> 127.0.0.1)
out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)
cluster = filter(lambda x: x['cluster_name'] == 'network_isolation_cluster_2', config.clusters)[0]
util.log(util.json_to_str(cluster))
# MGMT
mgmt_ip = cluster['servers'][0]['real_ip']
mgmt_port = cluster['servers'][0]['cm_port']
# Create cluster
ret = default_cluster.initialize_starting_up_smr_before_redis( cluster )
self.assertEqual(0, ret, 'failed to TestMaintenance.initialize')
# Place master on virtual ip address in order to cause master election.
pg_id = 0
m = util.get_server_by_role_and_pg(cluster['servers'], 'master', pg_id)
s = util.get_server_by_role_and_pg(cluster['servers'], 'slave', pg_id)
if m.has_key('ip') == True and m.has_key('real_ip') == False:
ret = util.role_change(cluster['servers'][0], cluster['cluster_name'], s['id'])
self.assertNotEquals(ret, -1, 'change %d to a master fail' % s['id'])
# Print initial state of cluster
util.log('\n\n\n ### INITIAL STATE OF CLUSTER ### ')
initial_state = []
self.assertTrue(util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, initial_state, check_quorum=True), 'failed to check cluster state')
# Network isolation test
for cnt in range(3):
# Block network
util.log('\n\n\n ### BLOCK NETWORK, %d ### ' % cnt)
out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)
for i in range(4):
util.log('waiting... %d' % (i + 1))
time.sleep(1)
# Check cluster state
ok = False
for i in range(7):
isolated_states = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, isolated_states, check_quorum=True)
time.sleep(1)
state_transition_done = True
for s in isolated_states:
if s['pgs_id'] == 1:
continue
if s['active_role'] != '?' or s['mgmt_role'] != 'N':
state_transition_done = False
if state_transition_done :
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state transition')
# Unblock network
util.log('\n\n\n ### UNBLOCK NETWORK, %d ### ' % cnt)
out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)
# Check cluster state
ok = False
for i in range(7):
final_state = []
util.check_cluster(cluster['cluster_name'], mgmt_ip, mgmt_port, final_state, check_quorum=True)
state_consistency = True
for s in final_state:
if s['pgs_id'] == 1:
continue
if s['active_role'] != s['mgmt_role']:
state_consistency = False
if state_consistency:
ok = True
break
time.sleep(1)
self.assertTrue(ok, 'Fail, state consistency')
# Check state
self.assertNotEqual(initial_state, None, 'initial_state is None')
self.assertNotEqual(final_state, None, 'final_state is None')
initial_state = sorted(initial_state, key=lambda x: int(x['pgs_id']))
final_state = sorted(final_state, key=lambda x: int(x['pgs_id']))
for i in range(len(final_state)):
msg = 'ts (%d)%d -> (%d)%d' % (initial_state[i]['pgs_id'], initial_state[i]['active_ts'], final_state[i]['pgs_id'], final_state[i]['active_ts'])
util.log(msg)
self.assertNotEqual(initial_state[i]['active_ts'], final_state[i]['active_ts'], msg)
# Shutdown cluster
ret = default_cluster.finalize( cluster )
self.assertEqual(ret, 0, 'failed to TestMaintenance.finalize')
# Delete forwarding role (127.0.0.100 -> 127.0.0.1)
out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)
out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
self.assertTrue(out.succeeded, 'delete a forwarding role to iptables fail. output:%s' % out)