self.assertTrue(check_manifest_sent)
def test_sendsnapshot_speed_without_limit(self):
"""
限速测试，stream_bandwidth_limit = 0，10k左右文件会立即发送成功
:return:
"""
self.update_conf(self.leaderpath, 'stream_bandwidth_limit', 0)
self.stop_client(self.leader)
time.sleep(5)
self.start_client(self.leader)
tname = self.now()
self.create(self.leader, tname, self.tid, self.pid, 144000, 2, "true")
self.put_data(self.leader, self.tid, self.pid, 100)
self.assertIn("MakeSnapshot ok", self.makesnapshot(self.leader, self.tid, self.pid))
self.assertIn("PauseSnapshot ok", self.pausesnapshot(self.leader, self.tid, self.pid))
self.assertIn("SendSnapshot ok", self.sendsnapshot(self.leader, self.tid, self.pid, self.slave1))
time.sleep(1)
self.check_manifest(self.slave1path, self.tid, self.pid,
'100', self.get_sdb_name(self.tid, self.pid), '100')
# Teardown
self.update_conf(self.leaderpath, 'stream_bandwidth_limit', None)
self.stop_client(self.leader)
time.sleep(1)
self.start_client(self.leader)
time.sleep(10)
if __name__ == "__main__":
load(TestSendSnapshot)
type_int64='int64:index',
type_uint32='uint32:index',
type_uint64='uint64:index',
type_double='double',
type_float='float',
type_string='string:index',
type_timestamp='timestamp:index',
type_date='date:index',
type_int16='int16:index',
type_uint16='uint16:index',
type_bool='bool:index'
)
self.assertIn('Create table ok' ,rs1)
(schema, column_key) = self.showschema(self.leader, self.tid, self.pid)
self.assertEqual(len(column_key), 0)
self.assertEqual(schema[0], ['0', 'type_string', 'string', 'yes'])
self.assertEqual(schema[1], ['1', 'type_bool', 'bool', 'yes'])
self.assertEqual(schema[2], ['2', 'type_int64', 'int64', 'yes'])
self.assertEqual(schema[3], ['3', 'type_uint64', 'uint64', 'yes'])
self.assertEqual(schema[4], ['4', 'type_timestamp', 'timestamp', 'yes'])
self.assertEqual(schema[5], ['5', 'type_int16', 'int16', 'yes'])
self.assertEqual(schema[6], ['6', 'type_double', 'double', 'no'])
self.assertEqual(schema[7], ['7', 'type_uint16', 'uint16', 'yes'])
self.assertEqual(schema[8], ['8', 'type_date', 'date', 'yes'])
self.assertEqual(schema[9], ['9', 'type_int32', 'int32', 'yes'])
self.assertEqual(schema[10], ['10', 'type_float', 'float', 'no'])
self.assertEqual(schema[11], ['11', 'type_uint32', 'uint32', 'yes'])
if __name__ == "__main__":
load(TestShowSchema)
time.sleep(1)
rs2 = self.offlineendpoint(self.ns_leader, self.leader)
self.assertIn("offline endpoint ok", rs2)
time.sleep(10)
rs3 = self.showtable(self.ns_leader, name)
self.assertEqual(rs3[(name, tid, '0', self.leader)],
['leader', '144000min', 'no', 'kNoCompress'])
self.assertEqual(rs3[(name, tid, '1', self.leader)],
['leader', '144000min', 'no', 'kNoCompress'])
self.assertEqual(rs3[(name, tid, '2', self.leader)],
['leader', '144000min', 'no', 'kNoCompress'])
self.assertEqual(rs3[(name, tid, '3', self.leader)],
['leader', '144000min', 'no', 'kNoCompress'])
self.assertEqual(rs3[(name, tid, '1', self.slave1)],
['leader', '144000min', 'yes', 'kNoCompress'])
self.assertEqual(rs3[(name, tid, '3', self.slave2)],
['leader', '144000min', 'yes', 'kNoCompress'])
self.assertEqual(rs3[(name, tid, '4', self.leader)],
['follower', '144000min', 'no', 'kNoCompress'])
self.assertIn(
'v2', self.scan(self.slave1, tid, 1, {'k1': 'pk1'}, self.now(), 1))
self.assertIn(
'v5', self.scan(self.slave1, tid, 2, {'k1': 'pk2'}, self.now(), 1))
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestOfflineEndpoint)
"ttl": 14400,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false"}
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 4)
self.assertEqual(schema[0], ["0", "card", "string", "yes"])
self.assertEqual(schema[1], ["1", "mcc", "string", "yes"])
self.assertEqual(schema[2], ["2", "amt", "double", "no"])
self.assertEqual(schema[3], ["3", "ts1", "int64", "no"])
self.ns_add_table_field(self.ns_leader, name, 'aa', 'string');
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 5)
self.assertEqual(schema[0], ["0", "card", "string", "yes"])
self.assertEqual(schema[1], ["1", "mcc", "string", "yes"])
self.assertEqual(schema[2], ["2", "amt", "double", "no"])
self.assertEqual(schema[3], ["3", "ts1", "int64", "no"])
self.assertEqual(schema[4], ["4", "aa", "string", "no"])
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestAddTableField)
rs = self.put(self.leader, tid, 1, '', self.now(), 'pk2' ,'v2', 'w2')
self.assertIn('Put ok', rs)
time.sleep(1)
self.stop_client(self.leader)
time.sleep(10)
self.offlineendpoint(self.ns_leader, self.leader)
self.start_client(self.leader)
time.sleep(3)
self.recoverendpoint(self.ns_leader, self.leader, 'true')
time.sleep(10)
rs5 = self.showtable(self.ns_leader, name)
self.assertEqual(rs5[(name, tid, '0', self.leader)], ['leader', '144000min', 'yes', 'kNoCompress'])
self.assertEqual(rs5[(name, tid, '1', self.leader)], ['leader', '144000min', 'yes', 'kNoCompress'])
self.assertEqual(rs5[(name, tid, '3', self.leader)], ['follower', '144000min', 'yes', 'kNoCompress'])
rs = self.put(self.leader, tid, 0, '', self.now(), 'pk3' ,'v3', 'w5')
self.assertIn('Put ok', rs)
rs = self.put(self.leader, tid, 1, '', self.now(), 'pk4' ,'v4', 'w4')
self.assertIn('Put ok', rs)
time.sleep(1)
self.assertIn('v1', self.scan(self.slave1, tid, 0, {'k1': 'pk1'}, self.now(), 1))
self.assertIn('v4', self.scan(self.slave1, tid, 1, {'k1': 'pk4'}, self.now(), 1))
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestRecoverEndpoint)
'amt': ('double', 1.1)}
rs4 = self.put(self.leader,
self.tid,
self.pid,
'testkey100',
self.now() - 1,
'testvalue100')
self.assertIn('Put ok', rs4)
rs5 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs5)
time.sleep(1)
mf = self.get_manifest(self.leaderpath, self.tid, self.pid)
self.assertEqual(mf['offset'], str(offset + 1))
# 新节点loadtable
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
rs6 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs6)
for i in range(0, offset):
self.multidimension_scan_vk = {'card': 'testkey{}'.format(i)} # for multidimension test
self.assertIn('testvalue', self.scan(
self.slave1, self.tid, self.pid, 'testkey{}'.format(i), self.now(), 1))
self.multidimension_scan_vk = {'card': 'testkey100'} # for multidimension test
self.assertIn('testvalue100', self.scan(self.slave1, self.tid, self.pid, 'testkey100', self.now(), 1))
if __name__ == "__main__":
load(TestMakeSnapshot)
)
@ddt.unpack
def test_create_compressed_table(self, exp_msg, *column_descs):
"""
column_desc参数测试
:param exp_msg:
:param column_descs:
:return:
"""
metadata_path = '{}/metadata.txt'.format(self.testpath)
m = utils.gen_table_metadata(
'"tname{}"'.format(time.time()), '"kAbsoluteTime"', 144000, 8,
('table_partition', '"{}"'.format(self.leader), '"0"', 'true'),
('table_partition', '"{}"'.format(self.slave1), '"0"', 'false'),
('table_partition', '"{}"'.format(self.slave2), '"0"', 'false'),
*column_descs)
m[0].append(("compress_type", '"snappy"'))
utils.gen_table_metadata_file(m, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
infoLogger.info(rs)
self.assertIn(exp_msg, rs)
if exp_msg == 'Create table ok':
rs1 = self.showtable(self.ns_leader)
infoLogger.info(rs1)
for v in rs1.values():
self.assertEqual(v[3], "kSnappy")
if __name__ == "__main__":
load(TestCreateTableByNsClient)
"""
健康的节点，状态为kTabletHealthy
:return:
"""
rs1 = self.showtablet(self.ns_leader)
infoLogger.info(rs1)
self.assertEqual(rs1[self.leader][0], 'kTabletHealthy')
self.assertEqual(rs1[self.slave1][0], 'kTabletHealthy')
self.assertEqual(rs1[self.slave2][0], 'kTabletHealthy')
def test_showtablet_offline(self):
"""
挂掉的节点，状态为kTabletOffline，启动后恢复为kTabletHealthy
:return:
"""
self.stop_client(self.slave1)
time.sleep(10)
rs1 = self.showtablet(self.ns_leader)
infoLogger.info(rs1)
self.assertEqual(rs1[self.leader][0], 'kTabletHealthy')
self.assertEqual(rs1[self.slave1][0], 'kTabletOffline')
self.assertEqual(rs1[self.slave2][0], 'kTabletHealthy')
self.start_client(self.slave1)
time.sleep(5)
rs2 = self.showtablet(self.ns_leader)
self.assertEqual(rs2[self.slave1][0], 'kTabletHealthy')
if __name__ == "__main__":
load(TestShowTablet)
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
for i in range(100):
rs = self.ns_put_multi_with_pair(self.ns_leader,name, ['card' + str(i%10), str(i+1), str(i+1), str(i+1)])
self.assertIn('Put ok', rs)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '0', 'ts1', '0', '8')
self.assertEqual(len(prs), 10)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '50', 'ts1', '0', '0')
self.assertEqual(len(prs), 5)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '50', 'ts1', '0', '8')
self.assertEqual(len(prs), 8)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '50', '0', 'ts1', '0', '8')
self.assertEqual(len(prs), 5)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '10', 'ts1', '8', '3')
self.assertEqual(len(prs), 8)
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '90', '0', 'ts1', '4', '5')
self.assertEqual(len(prs), 0)
if __name__ == "__main__":
load(TestHasTsCol)
'*', self.slave2, 'no')
self.assertIn('update ok', rs_absolute2)
self.assertIn('update ok', rs_absolute3)
rs_absolute2 = self.ns_update_table_alive_cmd(self.ns_leader,
'updatetablealive', 't1',
'*', self.slave1, 'yes')
rs_absolute3 = self.ns_update_table_alive_cmd(self.ns_leader,
'updatetablealive', 't1',
'*', self.slave2, 'yes')
self.assertIn('update ok', rs_absolute2)
self.assertIn('update ok', rs_absolute3)
rs_absolute2 = self.ns_update_table_alive_cmd(self.ns_leader,
'updatetablealive',
'tt1', '*', self.slave1,
'no')
rs_absolute3 = self.ns_update_table_alive_cmd(self.ns_leader,
'updatetablealive', 't1',
'10', self.slave2, 'no')
self.assertIn(
'Fail to update table alive. error msg: table is not exist',
rs_absolute2)
self.assertIn(
'Fail to update table alive. error msg: no pid has update',
rs_absolute3)
if __name__ == "__main__":
load(TestUpdateTableAlive)
"index_name": "card",
"ts_name": ["ts1", "ts2"]
},
{
"index_name": "mcc",
"ts_name": ["ts2"]
},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(column_key), 3)
self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "10"])
self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "100"])
self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "100"])
rs1 = self.ns_setttl(self.ns_leader, 'setttl', name, 'latest', 1,
'ts2')
self.assertIn('Set ttl ok !', rs1)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(column_key), 3)
self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "10"])
self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "1"])
self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "1"])
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestSetTTL)
rs1 = self.ns_put_multi(self.ns_leader, name, 11,
['card0', 'mcc0', '15'])
self.assertIn('Put ok', rs1)
rs2 = self.ns_put_multi(self.ns_leader, name, 22,
['card0', 'mcc1', '16'])
self.assertIn('Put ok', rs2)
rs3 = self.ns_put_multi(self.ns_leader, name, 33,
['card1', 'mcc2', '20'])
self.assertIn('Put ok', rs3)
rs4 = self.ns_preview(self.ns_leader, name, 2)
self.assertEqual(2, len(rs4))
self.assertEqual(rs4[0]['k1'], 'card1')
self.assertEqual(rs4[0]['k2'], 'mcc2')
self.assertEqual(rs4[0]['k3'], '20')
self.assertEqual(rs4[1]['k1'], 'card0')
self.assertEqual(rs4[1]['k2'], 'mcc1')
rs5 = self.ns_preview(self.ns_leader, name)
self.assertEqual(3, len(rs5))
self.assertEqual(rs5[0]['k1'], 'card1')
self.assertEqual(rs5[0]['k2'], 'mcc2')
self.assertEqual(rs5[0]['k3'], '20')
self.assertEqual(rs5[1]['k1'], 'card0')
self.assertEqual(rs5[1]['k2'], 'mcc1')
self.assertEqual(rs5[2]['k1'], 'card0')
self.assertEqual(rs5[2]['k2'], 'mcc0')
if __name__ == "__main__":
load(TestPreview)
('Fail to delreplica', '1,2,10', conf.tb_endpoints[1]),
('pid group[1,x,5] format error', '1,x,5', conf.tb_endpoints[1]),
('pid group[1,3:5] format error', '1,3:5', conf.tb_endpoints[1]),
('Fail to delreplica', '1-10', conf.tb_endpoints[1]),
('pid group[1~10] format error', '1~10', conf.tb_endpoints[1]),
('pid group[1-m] format error', '1-m', conf.tb_endpoints[1]),
('pid group[m-5] format error', 'm-5', conf.tb_endpoints[1]),
('Fail to delreplica', '5-7', conf.tb_endpoints[1]),
('Fail to delreplica', '5,6,7', conf.tb_endpoints[1]),
)
@ddt.unpack
def test_delreplica_pid_group_error(self, exp_msg, pid_group, endpoint):
"""
删除失败
:return:
"""
name = 't{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
m = utils.gen_table_metadata('"{}"'.format(name), '"kLatestTime"', 100, 8,
('table_partition', '"{}"'.format(self.leader), '"0-7"', 'true'),
('table_partition', '"{}"'.format(self.slave1), '"0-5"', 'false'))
utils.gen_table_metadata_file(m, metadata_path)
rs1 = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs1)
rs2 = self.delreplica(self.ns_leader, name, pid_group, 'ns_client', endpoint)
self.assertIn(exp_msg, rs2)
if __name__ == "__main__":
load(TestDelReplicaNs)
pid = tables.keys()[0][2]
table_endpoints = set()
table_endpoints.add(tables.keys()[0][3])
table_endpoints.add(tables.keys()[1][3])
replica_endpoint = endponints - table_endpoints
slave = replica_endpoint.pop()
self.ns_addreplica(self.ns_leader, name, pid, slave)
for i in range(10):
time.sleep(2)
rs = self.showopstatus(self.ns_leader, name, pid)
op_id = rs.keys()[0]
ops = self.showopstatus(self.ns_leader)
if ops[int(op_id)][1] != 'kDone':
continue
rs = self.gettablestatus(slave, tid, pid)
table_status = self.parse_tb(rs, ' ', [0, 1, 2, 3], [4, 5, 6])
rs = self.showtable_with_tablename(self.ns_leader, name)
table_infos = self.parse_tb(rs, ' ', [0, 1, 2, 3],
[4, 5, 6, 7, 8, 9, 10])
for table_info in table_infos:
self.assertEqual(table_status.keys()[0][2],
table_infos[table_info][4])
break
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestAddReplicaNs)
('column_desc', '"k2"', '"string"', 'false'),
('column_desc', '"k3"', '"string"', 'false')
)
utils.gen_table_metadata_file(m, metadata_path)
rs0 = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs0)
self.multidimension_vk = {'k1': ('string:index', 'testvalue0'),
'k2': ('string', 'testvalue0'),
'k3': ('string', 'testvalue0')}
self.multidimension_scan_vk = {'k1': 'testvalue0'}
rs1 = self.showtable(self.ns_leader)
tid = rs1.keys()[0][1]
self.assertEqual(rs1[(name, tid, str(pid), self.leader)], ['leader', '144000min', 'yes', 'kNoCompress'])
self.assertEqual(rs1[(name, tid, str(pid), self.slave1)], ['follower', '144000min', 'yes', 'kNoCompress'])
rs0 = self.changeleader(self.ns_leader, name, pid, switch)
time.sleep(1)
if (switch == ''):
self.assertIn(rs0, 'failed to change leader. error msg: leader is alive')
else:
rs2 = self.showtable(self.ns_leader)
self.assertEqual(rs2[(name, tid, str(pid), self.leader)], ['leader', '144000min', rsp_msg, 'kNoCompress'])
self.assertEqual(rs2[(name, tid, str(pid), conf.tb_endpoints[1])], ['leader', '144000min', 'yes', 'kNoCompress'])
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestChangeLeader)
self.assertEqual(column_key[0],
["0", "card_x", "card", "-", "14400min"])
self.assertEqual(column_key[1],
["1", "col", "col1|col2", "-", "14400min"])
row = ['card0', 'mcc0', '1.3', 'col1x', 'col2x']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
row = ['card0', 'mcc1', '1.4', 'col1m', 'col2m']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
row = ['card1', 'mcc1', '1.5', 'col11', 'col22']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card_x',
self.now() + 100, 0)
self.assertEqual(len(rs), 2)
rs5 = self.ns_delete(self.ns_leader, name, 'card0', 'card_x')
self.assertIn('delete ok', rs5)
rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card_x',
self.now() + 100, 0)
self.assertEqual(len(rs), 0)
rs = self.ns_scan_multi(self.ns_leader, name, 'col1m|col2m', 'col',
self.now() + 100, 0)
self.assertEqual(len(rs), 1)
rs5 = self.ns_delete(self.ns_leader, name, 'col1m|col2m', 'col')
self.assertIn('delete ok', rs5)
rs = self.ns_scan_multi(self.ns_leader, name, 'col1m|col2m', 'col',
self.now() + 100, 0)
self.assertEqual(len(rs), 0)
if __name__ == "__main__":
load(TestDelete)
m = utils.gen_table_metadata(
'"{}"'.format(self.tname), '"kAbsoluteTime"', 144000, 8,
('table_partition', '"{}"'.format(self.leader), '"0-3"', 'true'),
('table_partition', '"{}"'.format(self.slave1), '"2-3"', 'false'),
('table_partition', '"{}"'.format(self.slave2), '"2-3"', 'false'),
('column_desc', '"k1"', '"string"', 'true'),
('column_desc', '"k2"', '"string"', 'false'),
('column_desc', '"k3"', '"string"', 'false'))
utils.gen_table_metadata_file(m, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
table_info = self.showtable(self.ns_leader, self.tname)
self.tid = int(table_info.keys()[0][1])
self.pid = 1
for _ in range(10):
self.put(self.leader, self.tid, self.pid, 'testkey0',
self.now() + 90000, 'testvalue0')
steps_dict = self.get_steps_dict()
for i in steps:
eval(steps_dict[i])
rs = self.showtable(self.ns_leader, self.tname)
self.assertEqual(
rs[(self.tname, str(self.tid), str(self.pid), self.leader)],
['leader', '144000min', 'yes', 'kNoCompress'])
self.ns_drop(self.ns_leader, self.tname)
if __name__ == "__main__":
load(TestAutoRecoverTable)
self.assertEqual(table_status[:6], ['0', 'kTableLeader', 'kSnapshotPaused', 'true', '144000min', '0s'])
def test_changerole_to_leader_can_addreplica(self):
"""
切换从节点为主节点后，可以成功addreplica slave，slave可以同步leader数据
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid, 144000, 2, 'false')
self.assertIn('Create table ok', rs1)
rs2 = self.create(self.slave1, 't', self.tid, self.pid, 144000, 8, 'false')
self.assertIn('Create table ok', rs2)
rs3 = self.addreplica(self.leader, self.tid, self.pid, 'client', self.slave1)
self.assertIn('Fail to Add Replica', rs3)
rs4 = self.changerole(self.leader, self.tid, self.pid, 'leader')
self.assertIn('ChangeRole ok', rs4)
rs5 = self.addreplica(self.leader, self.tid, self.pid, 'client', self.slave1)
self.assertIn('AddReplica ok', rs5)
rs6 = self.put(self.leader,
self.tid,
self.pid,
'testkey0',
self.now(),
'testvalue0')
self.assertIn('Put ok', rs6)
self.assertIn('testvalue0', self.scan(self.slave1, self.tid, self.pid, 'testkey0', self.now(), 1))
if __name__ == "__main__":
load(TestChangeRole)
self.assertEqual(rs0[0], rs5[0])
self.assertEqual(rs0[0], rs6[0])
self.ns_drop(self.ns_leader, tname)
def test_ns_client_migrate_no_leader(self):
"""
无主状态下迁移失败
:return:
"""
tname = str(time.time())
self.createtable_put(tname, 500)
self.stop_client(self.leader)
time.sleep(2)
rs1 = self.migrate(self.ns_leader, self.slave1, tname, "4-6",
self.slave2)
time.sleep(2)
rs2 = self.showtable(self.ns_leader, tname)
self.start_client(self.leader)
time.sleep(10)
self.showtable(self.ns_leader, tname)
self.assertIn('partition migrate ok', rs1)
for i in range(4, 7):
self.assertIn((tname, str(self.tid), str(i), self.slave1), rs2)
self.assertNotIn((tname, str(self.tid), str(i), self.slave2), rs2)
self.ns_drop(self.ns_leader, tname)
if __name__ == "__main__":
load(TestNameserverMigrate)
self.assertEqual(memused2 > memused, True)
def test_gettablestatus_memused_datacount(self):
"""
检查数据量变大时，mem使用量是否正常
:return:
"""
self.multidimension_vk = {
'card': ('string:index', 'pk0'),
'merchant': ('string:index', 'pk1'),
'amt': ('string', 'a' * 100)
}
rs = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('ok', rs)
self.put_large_datas(10, 1)
table_status = self.get_table_status(self.leader, self.tid, self.pid)
memused = float(table_status[6])
self.pid = self.pid + 1
rs = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('ok', rs)
self.put_large_datas(20, 1)
table_status = self.get_table_status(self.leader, self.tid, self.pid)
memused2 = float(table_status[6])
self.assertEqual(memused2 > memused, True)
if __name__ == "__main__":
load(TestGetTableStatus)
暂停主节点指定表的snapshot，仍可以put数据且被同步
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid, 144000, 8, 'true')
self.assertIn('Create table ok', rs1)
rs2 = self.create(self.slave1, 't', self.tid, self.pid, 144000, 8, 'false')
self.assertIn('Create table ok', rs2)
rs3 = self.addreplica(self.leader, self.tid, self.pid, 'client', self.slave1)
self.assertIn('AddReplica ok', rs3)
time.sleep(1)
rs4 = self.pausesnapshot(self.leader, self.tid, self.pid)
self.assertIn('PauseSnapshot ok', rs4)
rs5 = self.put(self.leader,
self.tid,
self.pid,
'',
self.now() - 1,
'v', '1.1', 'k')
self.assertIn('Put ok', rs5)
time.sleep(1)
self.multidimension_scan_vk = {'card': 'k'} # for multidimension test
self.assertIn('v', self.scan(self.slave1, self.tid, self.pid, 'k', self.now(), 1))
table_status = self.get_table_status(self.leader, self.tid, self.pid)
self.assertEqual(table_status[:6], ['1', 'kTableLeader', 'kSnapshotPaused', 'true', '144000min', '0s'])
if __name__ == "__main__":
load(TestPauseSnapshot)
self.wait_op_done(name)
rs_old_leader_put = self.put(self.leader, tid, pid, 'testkey1',
self.now(), 'testvalue1')
self.assertIn('Put ok', rs_old_leader_put)
time.sleep(1)
self.assertFalse('testvalue1' in self.scan(self.slave1, tid, pid,
'testkey1', self.now(), 1))
self.assertFalse('testvalue1' in self.scan(self.slave2, tid, pid,
'testkey1', self.now(), 1))
rs_new_leader_put = self.put(self.slave1, tid, pid, 'testkey2',
self.now(), 'testvalue2')
self.assertIn('Put ok', rs_new_leader_put)
time.sleep(2)
self.assertFalse('testvalue2' in self.scan(self.leader, tid, pid,
'testkey2', self.now(), 1))
self.assertTrue('testvalue2' in self.scan(self.slave2, tid, pid,
'testkey2', self.now(), 1))
self.ns_recover_table_cmd(self.ns_leader, 'recovertable', name, pid,
self.leader)
time.sleep(1)
self.wait_op_done(name)
time.sleep(1)
self.assertTrue('testvalue2' in self.scan(self.leader, tid, pid,
'testkey2', self.now(), 1))
if __name__ == "__main__":
load(TestRecoverTable)
class TestConfSetGet(TestCaseBase):
@ddt.data(
('false', 'true', 'set auto_failover ok', 'true'),
('false', 'TRUE', 'set auto_failover ok', 'true'),
('true', 'FALSE', 'set auto_failover ok', 'false'),
('true', 'FalsE', 'set auto_failover ok', 'false'),
('true', '0',
'failed to set auto_failover. error msg: invalid parameter', 'true'),
('true', 'FAlsee',
'failed to set auto_failover. error msg: invalid parameter', 'true'),
('true', 'true', 'set auto_failover ok', 'true'),
('true', 'false', 'set auto_failover ok', 'false'),
)
@ddt.unpack
def test_auto_failover_confset(self, pre_set, set_value, msg, get_value):
"""
:return:
"""
self.confset(self.ns_leader, 'auto_failover', pre_set)
rs = self.confset(self.ns_leader, 'auto_failover', set_value)
self.assertIn(msg, rs)
rs1 = self.confget(self.ns_leader, 'auto_failover')
self.assertIn(get_value, rs1)
if set_value == "true" or set_value == "TRUE":
self.confset(self.ns_leader, 'auto_failover', 'false')
if __name__ == "__main__":
load(TestConfSetGet)
self.now() + 100, 0)
self.assertEqual(len(rs), 2)
self.assertEqual(rs[0]['card'], 'card' + str(i))
self.assertEqual(rs[0]['mcc'], 'mcc' + str(i))
self.assertEqual(rs[0]['loc'], 'loc' + str(i))
self.assertEqual(rs[0]['add2'], "2add{}".format(i))
rs = self.ns_deleteindex(self.ns_leader, name, "loc")
self.assertIn('delete index ok', rs)
time.sleep(2)
rs = self.ns_addindex(self.ns_leader, name, "loc2", "loc")
self.assertIn('addindex ok', rs)
time.sleep(2)
self.wait_op_done(name)
time.sleep(2)
for i in range(20):
rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i),
'loc2', 0, 0)
self.assertEqual(len(rs), 2)
self.assertEqual(rs[0]['card'], 'card' + str(i))
self.assertEqual(rs[0]['mcc'], 'mcc' + str(i))
self.assertEqual(rs[0]['loc'], 'loc' + str(i))
self.assertEqual(rs[0]['add2'], "2add{}".format(i))
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestSchema)
('setlimit', 'Get', 1.5, 'Bad set limit format'),
('setlimit', 'Scan', 10, 'Set Limit ok'),
('setlimit', 'Scan', 0, 'Set Limit ok'),
('setlimit', 'Scan', 1, 'Set Limit ok'),
('setlimit', 'Scan', 2147483647, 'Set Limit ok'),
('setlimit', 'Scan', -1, 'Fail to set limit'),
('setlimit', 'Scan', 1.5, 'Bad set limit format'),
('setlimit', 'Server', 10, 'Set Limit ok'),
('setlimit', 'Server', 0, 'Set Limit ok'),
('setlimit', 'Server', 1, 'Set Limit ok'),
('setlimit', 'Server', 2147483647, 'Set Limit ok'),
('setlimit', 'Server', -1, 'Fail to set limit'),
('setlimit', 'Server', 1.5, 'Bad set limit format'),
('setLimit', 'Scan', 1, 'unsupported cmd')
)
@ddt.unpack
def test_set_max_concurrency(self, command, method, max_concurrency_limit, rsp_msg):
"""
修改并发限制的数值
:param self:
:param max_concurrency_limit:
:return:
"""
rs1 = self.ns_setlimit(self.leader, command, method, max_concurrency_limit)
self.assertIn(rsp_msg, rs1)
if __name__ == "__main__":
load(TestSetLimit)
time.sleep(1)
rs1 = self.loadtable(self.slave1, 't', self.tid, self.pid, 1, 8,
'false')
self.assertIn('Fail', rs1)
rs0 = self.create(self.slave1, 't', self.tid, self.pid, 1, 8, 'false')
self.assertIn('Create table ok', rs0)
rs2 = self.addreplica(self.leader, self.tid, self.pid, 'client',
self.slave1)
self.assertIn('AddReplica ok', rs2)
time.sleep(2)
self.assertIn(
'testvalue0',
self.scan(self.slave1, self.tid, self.pid, {'card': 'testkey'},
self.now() + 60000 * 6, 1))
self.assertIn(
'testvalue1',
self.scan(self.slave1, self.tid, self.pid, {'card': 'testkey'},
self.now() + 60000 * 6, 1))
time.sleep(62)
self.assertFalse('testvalue0' in self.scan(
self.slave1, self.tid, self.pid, {'card': 'testkey'},
self.now() + 60000 * 6, 1))
self.assertIn(
'testvalue1',
self.scan(self.slave1, self.tid, self.pid, {'card': 'testkey'},
self.now() + 60000 * 6, 1))
if __name__ == "__main__":
load(TestLoadTable)
self.now() - 1, 'value{}'.format(i))
self.assertIn('Put ok', rs_put)
rs = self.showtable_with_tablename(self.ns_leader, name)
rs_after = self.parse_tb(rs, ' ', [0, 1, 2, 3], [4, 5, 6, 7, 8, 9, 10])
for i in range(10):
time.sleep(2)
offset_number = 0
rs = self.showtable_with_tablename(self.ns_leader, name)
rs_after = self.parse_tb(rs, ' ', [0, 1, 2, 3],
[4, 5, 6, 7, 8, 9, 10])
for table_info in rs_after:
if rs_after[table_info][4] == '2':
offset_number = offset_number + 1
if offset_number == 9:
break
diff = 1
for table_info in rs_after:
if rs_after[table_info][4] == '0':
continue
self.assertEqual(
diff,
int(rs_after[table_info][4]) - int(rs_before[table_info][4]))
self.confset(self.ns_leader, 'auto_failover', 'false')
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestAutoFailover)
self.assertEqual(rs[0]['mcc'], 'mcc0')
rs = self.ns_scan_multi(self.ns_leader, name, 'mcc1', 'mcc', 0, 0)
self.assertEqual(len(rs), 2)
rs = self.ns_get_multi(self.ns_leader, name, 'mcc1', 'mcc', 0)
self.assertEqual(rs['card'], 'card1')
self.assertEqual(rs['mcc'], 'mcc1')
rs = self.ns_get_multi(self.ns_leader, name, 'card0', 'card', 0)
self.assertEqual(rs['card'], 'card0')
self.assertEqual(rs['mcc'], 'mcc1')
rs = self.ns_count(self.ns_leader, name, 'card0', 'card')
self.assertEqual("count: 2", rs)
rs = self.ns_count(self.ns_leader, name, 'mcc1', 'mcc')
self.assertEqual("count: 2", rs)
rs = self.ns_count(self.ns_leader, name, 'mcc0', 'mcc')
self.assertEqual("count: 1", rs)
rs = self.ns_preview(self.ns_leader, name)
self.assertEqual(len(rs), 3)
rs = self.ns_delete(self.ns_leader, name, "mcc1", "mcc")
rs = self.ns_count(self.ns_leader, name, 'mcc1', 'mcc')
self.assertEqual("count: 0", rs)
self.ns_drop(self.ns_leader, name)
if __name__ == "__main__":
load(TestPartitionKey)
for k, v in value_ts_scannable.items():
rs = self.get(self.leader, self.tid, self.pid, 'pk', k[1])
infoLogger.info(rs)
if v is True:
self.assertIn('|{}|'.format(k[0]), rs)
else:
self.assertNotIn('|{}|'.format(k[0]), rs)
# put again after the last gc
for i in put_value_ts:
# for multidimension test
self.multidimension_vk = {
'card': ('string:index', 'pk'),
'merchant': ('string:index', '|{}|'.format(i[0])),
'amt': ('double', 1.1)
}
for ts in range(100):
self.put(self.leader, self.tid, self.pid, 'pk', i[1] - ts,
'|{}|'.format(i[0]))
time.sleep(1)
for k, v in value_ts_scannable2.items():
rs = self.get(self.leader, self.tid, self.pid, 'pk', k[1])
infoLogger.info(rs)
if v is True:
self.assertIn('|{}|'.format(k[0]), rs)
else:
self.assertNotIn('|{}|'.format(k[0]), rs)
if __name__ == "__main__":
load(TestTtl)
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
curtime = int(time.time() * 1000)
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.1', '1', '11'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc1', '1.2', '2', '12'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc2', '1.3', '3', '13'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc3', '1.4', '4', '14'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.6', '5', '15'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.7', '6', '16'])
self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc1', '1.8', '7', '17'])
self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc1', '1.9', '8', '18'])
self.ns_put_multi(self.ns_leader, name, '', ['card3', 'mcc4', '2.0', '9', '19'])
rs3 = self.makesnapshot(self.ns_leader, name, 0, 'ns_client')
table_info = self.showtable(self.ns_leader, name)
tid = table_info.keys()[0][1]
tablet_endpoint = table_info.keys()[0][3]
self.execute_gc(tablet_endpoint, tid, '0')
rs3 = self.makesnapshot(self.ns_leader, name, '0', 'ns_client')
self.assertIn('MakeSnapshot ok', rs3)
mf = self.get_manifest(self.leaderpath, tid, 0)
self.assertEqual(mf['offset'], '9')
self.assertEqual(mf['count'], '7')
if __name__ == "__main__":
load(TestMakeSnapshotNsClient)
