def test_5_transfer_pgs_to_another_machine(self):
util.print_frame()
self.load_gen_list = {}
# get gateway info
ip, port = util.get_rand_gateway(self.cluster)
gw = gateway_mgmt.Gateway(self.cluster['servers'][0]['id'])
ret = gw.connect(ip, port)
self.assertEqual(ret, 0,
'failed to connect to gateway, %s:%d' % (ip, port))
# incrase master generation number
util.log('failover in order to increase master generation number.')
max = 0
for i in range(5):
key_base = 'key'
for i in range(max, max + 10000):
cmd = 'set %s%d %d\r\n' % (key_base, i, i)
gw.write(cmd)
res = gw.read_until('\r\n')
self.assertEquals(res, '+OK\r\n')
max = max + 10000
m = util.get_server_by_role(self.cluster['servers'], 'master')
util.log('failover pgs%d' % m['id'])
ret = util.failover(m, self.leader_cm)
self.assertTrue(ret, 'failed to failover pgs%d' % m['id'])
# start load generator
util.log("start load_generator")
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
self.load_gen_list[i] = load_generator.LoadGenerator(i, ip, port)
self.load_gen_list[i].start()
time.sleep(5) # generate load for 5 sec
util.log("started load_generator")
m, s1, s2 = util.get_mss(self.cluster)
servers = [m, s1, s2]
# bgsave
for s in servers:
ret = util.bgsave(s)
self.assertTrue(ret, 'failed to bgsave. pgs%d' % s['id'])
new_servers = [config.server4, config.server5]
# add new slaves
for s in new_servers:
util.log('delete pgs%d`s check point.' % s['id'])
util.del_dumprdb(s['id'])
ret = util.cluster_util_getdump(s['id'], m['ip'], m['redis_port'],
'dump.rdb', 0, 8191)
self.assertEqual(
True, ret,
'failed : util.cluster_util_getdump returns false, src=%s:%d dest_pgsid=%d'
% (m['ip'], m['redis_port'], s['id']))
ret = util.install_pgs(self.cluster,
s,
self.leader_cm,
0,
rm_ckpt=False)
self.assertEqual(
True, ret,
'failed : util.pgs_add returns false, pgsid=%d' % s['id'])
util.log('succeeeded : add a new slave, pgsid=%d' % s['id'])
# check consistency
ok = True
for j in range(self.max_load_generator):
if self.load_gen_list[j].isConsistent() == False:
ok = False
break
if not ok:
break
for server_to_del in servers:
for s in servers:
util.pingpong(s['ip'], s['smr_mgmt_port'])
for s in new_servers:
util.pingpong(s['ip'], s['smr_mgmt_port'])
self.__del_server(server_to_del)
util.log('succeeded : delete pgs%d' % server_to_del['id'])
new_m = util.get_server_by_role(new_servers, 'master')
new_s = util.get_server_by_role(new_servers, 'slave')
self.assertNotEqual(new_m, None, 'master is None.')
self.assertNotEqual(new_s, None, 'slave is None.')
for s in new_servers:
util.pingpong(s['ip'], s['smr_mgmt_port'])
time.sleep(5) # generate load for 5 sec
# check consistency of load_generator
for i in range(self.max_load_generator):
self.load_gen_list[i].quit()
for i in range(self.max_load_generator):
self.load_gen_list[i].join()
self.assertTrue(self.load_gen_list[i].isConsistent(),
'Inconsistent after migration')
self.load_gen_list.pop(i, None)
# Go back to initial configuration
# recover pgs
for s in servers:
self.assertTrue(
util.install_pgs(self.cluster,
s,
self.leader_cm,
rm_ckpt=False),
'failed to recover pgs. (install_pgs)')
# cleanup new slaves
for s in new_servers:
self.assertTrue(
util.uninstall_pgs(self.cluster, s, self.leader_cm),
'failed to cleanup pgs. (uninstall_pgs)')
def test_5_transfer_pgs_to_another_machine(self):
util.print_frame()
self.load_gen_list = {}
# get gateway info
ip, port = util.get_rand_gateway( self.cluster )
gw = gateway_mgmt.Gateway( self.cluster['servers'][0]['id'] )
ret = gw.connect( ip, port )
self.assertEqual( ret, 0, 'failed to connect to gateway, %s:%d' % (ip, port) )
# incrase master generation number
util.log('failover in order to increase master generation number.')
max = 0
for i in range(5):
key_base = 'key'
for i in range(max, max+10000):
cmd = 'set %s%d %d\r\n' % (key_base, i, i)
gw.write( cmd )
res = gw.read_until( '\r\n' )
self.assertEquals( res, '+OK\r\n' )
max = max + 10000
m = util.get_server_by_role(self.cluster['servers'], 'master')
util.log('failover pgs%d' % m['id'])
ret = util.failover(m, self.leader_cm)
self.assertTrue(ret, 'failed to failover pgs%d' % m['id'])
# start load generator
util.log("start load_generator")
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
self.load_gen_list[i] = load_generator.LoadGenerator(i, ip, port)
self.load_gen_list[i].start()
time.sleep(5) # generate load for 5 sec
util.log("started load_generator")
m, s1, s2 = util.get_mss(self.cluster)
servers = [m, s1, s2]
# bgsave
for s in servers:
ret = util.bgsave(s)
self.assertTrue(ret, 'failed to bgsave. pgs%d' % s['id'])
new_servers = [config.server4, config.server5]
# add new slaves
for s in new_servers:
util.log('delete pgs%d`s check point.' % s['id'])
util.del_dumprdb(s['id'])
ret = util.cluster_util_getdump(s['id'], m['ip'], m['redis_port'], 'dump.rdb', 0, 8191)
self.assertEqual(True, ret,
'failed : util.cluster_util_getdump returns false, src=%s:%d dest_pgsid=%d' % (
m['ip'], m['redis_port'], s['id']))
ret = util.pgs_add(self.cluster, s, self.leader_cm, 0, rm_ckpt=False)
self.assertEqual(True, ret, 'failed : util.pgs_add returns false, pgsid=%d' % s['id'])
util.log('succeeeded : add a new slave, pgsid=%d' % s['id'])
# check consistency
ok = True
for j in range(self.max_load_generator):
if self.load_gen_list[j].isConsistent() == False:
ok = False
break
if not ok:
break;
for server_to_del in servers:
for s in servers:
util.pingpong( s['ip'], s['smr_mgmt_port'] )
for s in new_servers:
util.pingpong( s['ip'], s['smr_mgmt_port'] )
self.__del_server(server_to_del)
util.log('succeeded : delete pgs%d' % server_to_del['id'])
new_m = util.get_server_by_role(new_servers, 'master')
new_s = util.get_server_by_role(new_servers, 'slave')
self.assertNotEqual( new_m, None, 'master is None.' )
self.assertNotEqual( new_s, None, 'slave is None.' )
for s in new_servers:
util.pingpong( s['ip'], s['smr_mgmt_port'] )
time.sleep(5) # generate load for 5 sec
# check consistency of load_generator
for i in range(self.max_load_generator):
self.load_gen_list[i].quit()
for i in range(self.max_load_generator):
self.load_gen_list[i].join()
self.assertTrue(self.load_gen_list[i].isConsistent(), 'Inconsistent after migration')
self.load_gen_list.pop(i, None)
def test_1_role_change(self):
util.print_frame()
self.load_gen_list = {}
# Start load generator
util.log("Start load_generator")
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
# Loop (smr: 3 copy)
for i in range(30):
target_server = util.get_server_by_role(self.cluster['servers'],
'slave')
self.assertNotEquals(target_server, None, 'Get slave fail.')
target = target_server['id']
print ''
util.log("(3 copy) Loop:%d, target pgs:%d" % (i, target))
# Get old timestamp
util.log_server_state(self.cluster)
old_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs(s)
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm,
self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target + 1) % 3
util.log('Change role success.')
# Wait until role change finished
for s in self.cluster['servers']:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
try:
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
except:
pass
time.sleep(0.2)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state(self.cluster)
new_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs(s)
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(3):
self.assertNotEqual(
old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' %
(old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(),
'Data inconsistency after role_change')
# Loop (smr: 2 copy)
self.__del_server(self.cluster['servers'][0])
servers = [self.cluster['servers'][1], self.cluster['servers'][2]]
normal_state = False
for i in xrange(20):
normal_state = util.check_cluster(self.cluster['cluster_name'],
self.leader_cm['ip'],
self.leader_cm['cm_port'],
check_quorum=True)
if normal_state:
break
time.sleep(0.5)
self.assertTrue(normal_state, "Unstable cluster state")
for i in range(30):
print ''
util.log("(2 copy) Loop:%d, target pgs:%d" % (i, target))
s = util.get_server_by_role(servers, 'slave')
target = s['id']
# Get old timestamp
util.log_server_state(self.cluster)
old_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs(s)
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm,
self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target) % 2 + 1
util.log('Change role success.')
# Wait until role change finished
for s in servers:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
time.sleep(0.1)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state(self.cluster)
new_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs(s)
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(2):
self.assertNotEqual(
old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' %
(old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(),
'Data inconsistency after role_change')
# Go back to initial configuration
self.assertTrue(
util.install_pgs(self.cluster,
self.cluster['servers'][0],
self.leader_cm,
rm_ckpt=False), 'failed to recover pgs.')
def test_1_role_change(self):
util.print_frame()
self.load_gen_list = {}
# Start load generator
util.log("Start load_generator")
for i in range(self.max_load_generator):
ip, port = util.get_rand_gateway(self.cluster)
load_gen = load_generator.LoadGenerator(i, ip, port)
load_gen.start()
self.load_gen_list[i] = load_gen
# Loop (smr: 3 copy)
target_server = util.get_server_by_role(self.cluster['servers'], 'slave')
self.assertNotEquals(target_server, None, 'Get slave fail.')
target = target_server['id']
for i in range(30):
print ''
util.log("(3 copy) Loop:%d, target pgs:%d" % (i, target))
# Get old timestamp
util.log_server_state( self.cluster )
old_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs( s )
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm, self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target + 1) % 3
util.log('Change role success.')
# Wait until role change finished
for s in self.cluster['servers']:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
time.sleep(0.1)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state( self.cluster )
new_timestamp_list = []
for s in self.cluster['servers']:
ts = util.get_timestamp_of_pgs( s )
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(3):
self.assertNotEqual(old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' % (old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(), 'Data inconsistency after role_change')
# Loop (smr: 2 copy)
self.__del_server(self.cluster['servers'][0])
servers = [self.cluster['servers'][1], self.cluster['servers'][2]]
s = util.get_server_by_role(servers, 'slave')
target = s['id']
for i in range(30):
print ''
util.log("(2 copy) Loop:%d, target pgs:%d" % (i, target))
# Get old timestamp
util.log_server_state( self.cluster )
old_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs( s )
old_timestamp_list.append(ts)
# Role change
master = util.role_change(self.leader_cm, self.cluster['cluster_name'], target)
self.assertNotEqual(master, -1, 'role_change error.')
while target == master:
target = (target) % 2 + 1
util.log('Change role success.')
# Wait until role change finished
for s in servers:
max_try_cnt = 20
ok = False
for try_cnt in range(max_try_cnt):
pong = util.pingpong(s['ip'], s['redis_port'])
if pong != None and pong == '+PONG\r\n':
ok = True
break
time.sleep(0.1)
self.assertTrue(ok, 'redis state error.')
# Get new timestamp
util.log_server_state( self.cluster )
new_timestamp_list = []
for s in servers:
ts = util.get_timestamp_of_pgs( s )
new_timestamp_list.append(ts)
# Compare old timestamps and new timestamps
for i in range(2):
self.assertNotEqual(old_timestamp_list[i], new_timestamp_list[i],
'Timestamp is not changed. %d->%d' % (old_timestamp_list[i], new_timestamp_list[i]))
# Cheeck Consistency
for load_gen_id, load_gen in self.load_gen_list.items():
self.assertTrue(load_gen.isConsistent(), 'Data inconsistency after role_change')