def create(self,
endpoint,
tname,
tid,
pid,
ttl=144000,
segment=8,
isleader='true',
*slave_endpoints,
**schema):
if not schema:
if self.multidimension:
infoLogger.debug('create with default multi dimension')
cmd = 'screate'
schema = {k: v[0]
for k, v in self.multidimension_vk.items()
} # default schema
else:
cmd = 'create'
else:
cmd = 'screate'
return self.run_client(
endpoint, '{} {} {} {} {} {} {} {} {}'.format(
cmd, tname, tid, pid, ttl, segment, isleader,
' '.join(slave_endpoints), ' '.join([
'{}:{}'.format(k, v) for k, v in schema.items() if k != ''
])))
def scan(self, endpoint, tid, pid, vk, ts_from, ts_to):
"""
:param endpoint:
:param tid:
:param pid:
:param vk: e.g. {'card': 0001, 'merchant': 0002} or 'naysakey'
:param ts_from:
:param ts_to:
:return:
"""
if not isinstance(vk, dict):
if self.multidimension:
infoLogger.debug('scan with default multi dimension')
default_vk = self.multidimension_scan_vk
value_key = [
'{} {}'.format(v, k) for k, v in default_vk.items()
]
return self.run_client(
endpoint,
'sscan {} {} {} {} {}'.format(tid, pid,
' '.join(value_key), ts_from,
ts_to))
else:
return self.run_client(
endpoint,
'scan {} {} {} {} {}'.format(tid, pid, vk, ts_from, ts_to))
else:
value_key = ['{} {}'.format(v, k) for k, v in vk.items()]
return self.run_client(
endpoint,
'sscan {} {} {} {} {}'.format(tid, pid, ' '.join(value_key),
ts_from, ts_to))
def ns_count(self,
endpoint,
name,
key,
idx_name,
filter_expired_data='false'):
cmd = 'count {} {} {} {}'.format(name, key, idx_name,
filter_expired_data)
result = self.run_client(endpoint, cmd, 'ns_client')
infoLogger.debug(result)
return result
def find_new_tb_leader(self, tname, tid, pid):
rs = self.showtable(self.ns_leader, tname)
infoLogger.info(rs)
for (key, value) in rs.items():
if key[1] == str(tid) and key[2] == str(pid):
infoLogger.info(value)
if value[0] == "leader" and value[2] == "yes":
new_leader = key[3]
infoLogger.debug('------new leader:' + new_leader +
'-----------')
self.new_tb_leader = new_leader
return new_leader
def get_latest_opid_by_tname_pid(self, tname, pid):
rs = self.run_client(self.ns_leader,
'showopstatus {} {}'.format(tname,
pid), 'ns_client')
opstatus = self.parse_tb(rs, ' ', [0], [1, 4, 8])
op_id_arr = []
for op_id in opstatus.keys():
op_id_arr.append(int(op_id))
self.latest_opid = sorted(op_id_arr)[-1]
infoLogger.debug('------latest_opid:' + str(self.latest_opid) +
'---------------')
return self.latest_opid
def ns_count_with_pair(self,
endpoint,
name,
key,
idx_name,
ts_name,
filter_expired_data='false'):
cmd = 'count {} {} {} {} {}'.format(
'table_name=' + name, 'key=' + key, 'index_name=' + idx_name,
'ts_name=' + ts_name, 'filter_expired_data=' + filter_expired_data)
result = self.run_client(endpoint, cmd, 'ns_client')
infoLogger.debug(result)
return result
def put(self, endpoint, tid, pid, key, ts, *values):
if len(values) == 1:
if self.multidimension and key is not '':
infoLogger.debug('put with default multi dimension')
default_values = [
str(v[1]) for v in self.multidimension_vk.values()
]
return self.run_client(
endpoint,
'sput {} {} {} {}'.format(tid, pid, ts,
' '.join(default_values)))
elif not self.multidimension and key is not '':
return self.run_client(
endpoint,
'put {} {} {} {} {}'.format(tid, pid, key, ts, values[0]))
return self.run_client(
endpoint, 'sput {} {} {} {}'.format(tid, pid, ts,
' '.join(values)))
def test_addreplica_check_binlog_sync_progress(self):
"""
测试binlog在不同阈值的时候，数据追平之后，添加备份的状态是否为yes
:return:
"""
name = 't{}'.format(time.time())
infoLogger.info(name)
endponints = self.get_tablet_endpoints()
rs1 = self.ns_create_cmd(self.ns_leader, name, 144000, 1, 2, '')
self.assertIn('Create table ok', rs1)
number = 200
for i in range(number):
rs_put = self.ns_put_kv(self.ns_leader, name, 'key{}'.format(i),
self.now() - 1, 'value{}'.format(i))
tables = self.showtable(self.ns_leader, name)
tid = tables.keys()[0][1]
pid = tables.keys()[0][2]
table_endpoints = set()
table_endpoints.add(tables.keys()[0][3])
table_endpoints.add(tables.keys()[1][3])
replica_endpoint = endponints - table_endpoints
slave = replica_endpoint.pop()
row = ''
self.ns_addreplica(self.ns_leader, name, pid, slave)
for repeat in range(10):
time.sleep(2)
rs = self.ns_showopstatus(self.ns_leader)
ops = self.parse_tb(rs, ' ', [0], [1, 2, 3, 4, 5, 6])
row = ''
infoLogger.debug('{}'.format(rs))
for status in ops:
if ops[status][1] == name and ops[status][3] == 'kDone':
row = status
break
if row != '':
break
self.assertIn('kDone', ops[row][3])
self.assertIn(name, ops[row][1])
self.ns_drop(self.ns_leader, name)
def test_auto_failover_kill_ns(self):
"""
auto_failover打开后，下线一个tablet和nameserver,在超时时间内，tablet自动恢复，leader和下线的节点的offset保持一致
:return:
"""
self.confset(self.ns_leader, 'auto_failover', 'true')
name = 't{}'.format(time.time())
infoLogger.info(name)
rs1 = self.ns_create_cmd(self.ns_leader, name, 144000, 1, 3, '')
self.assertIn('Create table ok', rs1)
number = 3
for i in range(number):
rs_put = self.ns_put_kv(self.ns_leader, name, 'key{}'.format(i),
self.now() - 1, 'value{}'.format(i))
self.assertIn('Put ok', rs_put)
rs = self.showtable_with_tablename(self.ns_leader, name)
rs = self.parse_tb(rs, ' ', [0, 1, 2, 3], [4, 5, 6, 7, 8, 9, 10])
tid = rs.keys()[0][1]
pid = rs.keys()[0][2]
self.print_table('', name)
rs_before = self.gettablestatus(self.slave1, tid, pid)
rs_before = self.parse_tb(rs_before, ' ', [0, 1, 2, 3],
[4, 5, 6, 7, 8, 9, 10])
for i in range(20):
rs_before = self.gettablestatus(self.slave1, tid, pid)
rs_before = self.parse_tb(rs_before, ' ', [0, 1, 2, 3],
[4, 5, 6, 7, 8, 9, 10])
if '{}'.format(rs_before) == 'gettablestatus failed':
time.sleep(2)
continue
break
infoLogger.info(rs_before)
self.assertFalse('gettablestatus failed' in '{}'.format(rs_before))
rs = self.ns_showopstatus(self.ns_leader)
self.stop_client(self.slave1)
time.sleep(2)
self.stop_client(self.ns_leader)
time.sleep(5)
self.start_client(self.slave1)
time.sleep(10)
rs_after = self.gettablestatus(self.slave1, tid, pid)
rs_after = self.parse_tb(rs_after, ' ', [0, 1, 2, 3],
[4, 5, 6, 7, 8, 9, 10])
for i in range(20):
time.sleep(2)
rs_after = self.gettablestatus(self.slave1, tid, pid)
rs_after = self.parse_tb(rs_after, ' ', [0, 1, 2, 3],
[4, 5, 6, 7, 8, 9, 10])
if '{}'.format(rs_after) == 'gettablestatus failed':
continue
if rs_before.keys()[0][2] == rs_after.keys()[0][2]:
self.assertIn(rs_before.keys()[0][2], rs_after.keys()[0][2])
break
rs = self.ns_showopstatus(self.ns_slaver)
if '{}'.format(rs_after) == 'gettablestatus failed':
infoLogger.debug('{}'.format(rs))
self.print_table(self.ns_slaver, '')
self.assertIn(rs_before.keys()[0][2], rs_after.keys()[0][2])
self.confset(self.ns_leader, 'auto_failover', 'false')
self.start_client(self.ns_leader, 'nameserver')
self.get_new_ns_leader()
self.ns_drop(self.ns_leader, name)
def ns_drop(self, endpoint, tname):
infoLogger.debug(tname)
return self.run_client(endpoint, 'drop {}'.format(tname), 'ns_client')
def test_changeleader_master_disconnect(self):
"""
changeleader功能正常，主节点断网后，可以手工故障切换，切换成功后从节点可以同步数据
:return:
"""
self.start_client(self.leader)
metadata_path = '{}/metadata.txt'.format(self.testpath)
name = 'tname{}'.format(time.time())
infoLogger.info(name)
m = utils.gen_table_metadata(
'"{}"'.format(name), None, 144000, 2,
('table_partition', '"{}"'.format(self.leader), '"0-2"', 'true'),
('table_partition', '"{}"'.format(self.slave1), '"0-1"', 'false'),
('table_partition', '"{}"'.format(self.slave2), '"0-2"', 'false'),
('column_desc', '"k1"', '"string"', 'true'),
('column_desc', '"k2"', '"string"', 'false'),
('column_desc', '"k3"', '"string"', 'false')
)
utils.gen_table_metadata_file(m, metadata_path)
rs0 = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs0)
self.multidimension_vk = {'k1': ('string:index', 'testvalue0'),
'k2': ('string', 'testvalue0'),
'k3': ('string', 'testvalue0')}
self.multidimension_scan_vk = {'k1': 'testvalue0'}
rs1 = self.showtable(self.ns_leader, name)
tid = rs1.keys()[0][1]
rs = self.disconnectzk(self.leader)
self.assertIn('disconnect zk ok', rs)
rs = self.updatetablealive(self.ns_leader, name, '*', self.leader, 'no')
self.assertIn('update ok', rs)
time.sleep(3)
rs = self.changeleader(self.ns_leader, name, 0, 'auto')
self.assertIn('change leader ok', rs)
time.sleep(2)
rs = self.connectzk(self.leader)
self.assertIn('connect zk ok', rs)
for repeat in range(20):
time.sleep(2)
rs = self.ns_showopstatus(self.ns_leader)
tablestatus = self.parse_tb(rs, ' ', [0, 1, 2, 3], [4, 5, 6, 7])
kDone_cnt = 0
cnt = 0
flag = False
for status in tablestatus:
if status[2] == name:
cnt = cnt + 1
infoLogger.info('{} = {}'.format(status, tablestatus[status]))
if tablestatus[status][0] == 'kFailed':
infoLogger.error('{} = {}'.format(status, tablestatus[status]))
flag = True
break
if tablestatus[status][0] == 'kDone':
infoLogger.info('{} = {}'.format(status, tablestatus[status]))
kDone_cnt = kDone_cnt + 1
if flag == True:
break
if kDone_cnt == cnt:
break
rs2 = self.showtable(self.ns_leader)
act1 = rs2[(name, tid, '0', self.slave1)]
act2 = rs2[(name, tid, '0', self.slave2)]
roles = [x[0] for x in [act1, act2]]
for repeat in range(20):
rs = self.showtable(self.ns_leader, name)
act1 = rs2[(name, tid, '0', self.slave1)]
act2 = rs2[(name, tid, '0', self.slave2)]
roles = [x[0] for x in [act1, act2]]
if roles.count('leader') == 1 and roles.count('follower') == 1:
self.assertEqual(roles.count('leader'), 1)
self.assertEqual(roles.count('follower'), 1)
break
time.sleep(2)
self.assertEqual(rs2[(name, tid, '0', self.leader)], ['leader', '144000min', 'no', 'kNoCompress'])
self.assertEqual(rs2[(name, tid, '1', self.leader)], ['leader', '144000min', 'no', 'kNoCompress'])
self.assertEqual(rs2[(name, tid, '2', self.leader)], ['leader', '144000min', 'no', 'kNoCompress'])
if roles.count('leader') != 1:
rs = self.ns_showopstatus(self.ns_leader)
infoLogger.debug(rs)
self.assertEqual(roles.count('leader'), 1)
self.assertEqual(roles.count('leader'), 1)
self.assertEqual(roles.count('follower'), 1)
leader_new = self.slave1 if 'leader' in act1 else self.slave2
follower = self.slave1 if 'follower' in act1 else self.slave2
rs2 = self.put(self.leader, tid, 1, 'testkey0', self.now(), 'testvalue0')
self.assertIn('Put ok', rs2)
rs3 = self.put(self.slave1, tid, 1, 'testkey0', self.now(), 'testvalue0')
self.assertIn('Put failed', rs3)
rs4 = self.put(leader_new, tid, 0, 'testkey0', self.now(), 'testvalue0')
self.assertIn('Put ok', rs4)
time.sleep(1)
self.assertIn('testvalue0', self.scan(follower, tid, 0, 'testkey0', self.now(), 1))
self.ns_drop(self.ns_leader, name)