def test_add_table_field_without_columnkey(self):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 14400,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false"}
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 4)
self.assertEqual(schema[0], ["0", "card", "string", "yes"])
self.assertEqual(schema[1], ["1", "mcc", "string", "yes"])
self.assertEqual(schema[2], ["2", "amt", "double", "no"])
self.assertEqual(schema[3], ["3", "ts1", "int64", "no"])
self.ns_add_table_field(self.ns_leader, name, 'aa', 'string');
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 5)
self.assertEqual(schema[0], ["0", "card", "string", "yes"])
self.assertEqual(schema[1], ["1", "mcc", "string", "yes"])
self.assertEqual(schema[2], ["2", "amt", "double", "no"])
self.assertEqual(schema[3], ["3", "ts1", "int64", "no"])
self.assertEqual(schema[4], ["4", "aa", "string", "no"])
self.ns_drop(self.ns_leader, name)
def test_create_showtable_record_cnt(self, exp_msg, *eles):
"""
使用文件column_key创建表，put数据后showtable检查record的数量
"""
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 120,
"partition_num": 3,
"replica_num": 3,
"column_desc": [],
"column_key": []
}
for item in eles:
table_meta[item[0]].append(item[1])
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
infoLogger.info(rs)
self.assertIn(exp_msg, rs)
self.ns_put_multi(self.ns_leader, name, 10, ['card0', '123', '1.1'])
self.ns_put_multi(self.ns_leader, name, 10, ['card1', '123', '2.2'])
self.ns_put_multi(self.ns_leader, name, 10, ['card1', '233', '3.3'])
self.ns_put_multi(self.ns_leader, name, 10, ['card0', '233', '4.4'])
time.sleep(2)
table_info = self.showtable_with_tablename(self.ns_leader, name)
table_info = self.parse_tb(table_info, ' ', [1, 2, 3], [9])
record_cnt = 0
for k, v in table_info.items():
record_cnt += int(v[0])
self.assertEqual(record_cnt, 12)
self.ns_drop(self.ns_leader, name)
def test_partitionkey_create(self, args):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
0,
"column_desc": [
{
"name": "col1",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "col2",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "col3",
"type": "double",
"add_ts_idx": "false"
},
],
"partition_key":
self.convert_partition_key(args[0])
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn(args[1], rs)
if args[1] == "Create table ok":
result = self.ns_info(self.ns_leader, name)
self.assertEqual(result["partition_key"].replace(" ", ""), args[0])
self.ns_drop(self.ns_leader, name)
def test_showschema_no_columnkey(self):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
14400,
"column_desc": [
{
"name": "card",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "mcc",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "amt",
"type": "double",
"add_ts_idx": "false"
},
{
"name": "ts1",
"type": "int64",
"add_ts_idx": "false",
"is_ts_col": "true"
},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 4)
self.assertEqual(schema[0], ["0", "card", "string"])
self.assertEqual(schema[2], ["2", "amt", "double"])
self.assertEqual(schema[3], ["3", "ts1", "int64"])
self.assertEqual(len(column_key), 2)
self.assertEqual(column_key[0],
["0", "card", "card", "ts1", "14400min"])
self.assertEqual(column_key[1], ["1", "mcc", "mcc", "ts1", "14400min"])
rs = self.ns_deleteindex(self.ns_leader, name, "card")
self.assertIn('Fail to delete index', rs)
rs = self.ns_deleteindex(self.ns_leader, name, "mcc")
self.assertIn('delete index ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 4)
self.assertEqual(schema[0], ["0", "card", "string"])
self.assertEqual(schema[2], ["2", "amt", "double"])
self.assertEqual(schema[3], ["3", "ts1", "int64"])
self.assertEqual(len(column_key), 1)
self.assertEqual(column_key[0],
["0", "card", "card", "ts1", "14400min"])
self.ns_drop(self.ns_leader, name)
def test_scan_schema_has_ts_col(self):
"""
指定时间列的schema表查询pk下的多条数据
:return:
"""
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 0,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
],
"column_key":[
{"index_name":"card", "ts_name":["ts1", "ts2"]},
{"index_name":"mcc", "ts_name":["ts2"]},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
rs1 = self.ns_put_multi_with_pair(self.ns_leader,name, ['card0', 'mcc0', '1', '15', '18'])
self.assertIn('Put ok', rs1)
rs2 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card0', 'mcc1', '2', '16', '19'])
self.assertIn('Put ok', rs2)
rs3 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card1', 'mcc2', '3', '20', '20'])
self.assertIn('Put ok', rs3)
rs4 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card0', 'card', '25', '0', 'ts1')
self.assertEqual(len(rs4),2)
self.assertEqual(rs4[0]['card'], 'card0')
self.assertEqual(rs4[0]['mcc'], 'mcc1')
self.assertEqual(rs4[0]['amt'], '2')
self.assertEqual(rs4[0]['ts1'], '16')
self.assertEqual(rs4[0]['ts2'], '19')
self.assertEqual(rs4[1]['card'], 'card0')
self.assertEqual(rs4[1]['mcc'], 'mcc0')
self.assertEqual(rs4[1]['amt'], '1')
self.assertEqual(rs4[1]['ts1'], '15')
self.assertEqual(rs4[1]['ts2'], '18')
rs5 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card0', 'card', '25', '0', 'ts1', '1')
self.assertEqual(len(rs5),1)
self.assertEqual(rs5[0]['card'], 'card0')
self.assertEqual(rs5[0]['mcc'], 'mcc1')
self.assertEqual(rs5[0]['amt'], '2')
self.assertEqual(rs5[0]['ts1'], '16')
self.assertEqual(rs5[0]['ts2'], '19')
rs6 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'mcc1', 'mcc', '25', '0', 'ts1', '1')
self.assertEqual(len(rs6), 0)
rs7 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'mcc1', 'mcc', '25', '0', 'ts2', '1')
self.assertEqual(len(rs7), 1)
def test_add_table_field_with_columnkey(self):
"""
指定时间列的schema表统计pk下的条数
:return:
"""
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 0,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
],
"column_key":[
{"index_name":"card", "ts_name":["ts1", "ts2"]},
{"index_name":"mcc", "ts_name":["ts2"]},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 5)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[1], ['1', 'mcc', 'string'])
self.assertEqual(schema[2], ['2', 'amt', 'double'])
self.assertEqual(schema[3], ['3', 'ts1', 'int64'])
self.assertEqual(schema[4], ['4', 'ts2', 'int64'])
self.assertEqual(len(column_key), 3)
self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "0min"])
self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "0min"])
self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "0min"])
self.ns_add_table_field(self.ns_leader, name, 'aa', 'string');
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 6)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[1], ['1', 'mcc', 'string'])
self.assertEqual(schema[2], ['2', 'amt', 'double'])
self.assertEqual(schema[3], ['3', 'ts1', 'int64'])
self.assertEqual(schema[4], ['4', 'ts2', 'int64'])
self.assertEqual(schema[5], ['5', 'aa', 'string'])
self.assertEqual(len(column_key), 3)
self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "0min"])
self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "0min"])
self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "0min"])
self.ns_drop(self.ns_leader, name)
def test_count_schema_has_ts_col(self):
"""
指定时间列的schema表统计pk下的条数
:return:
"""
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 0,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
],
"column_key":[
{"index_name":"card", "ts_name":["ts1", "ts2"]},
{"index_name":"mcc", "ts_name":["ts2"]},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
rs1 = self.ns_put_multi_with_pair(self.ns_leader,name, ['card0', 'mcc0', '1', '15', '18'])
self.assertIn('Put ok', rs1)
rs2 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card0', 'mcc1', '2', '16', '19'])
self.assertIn('Put ok', rs2)
rs3 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card1', 'mcc2', '3', '20', '20'])
self.assertIn('Put ok', rs3)
rs4 = self.ns_count_with_pair(self.ns_leader, name, 'card0', 'card', 'ts1')
self.assertIn('count: 2', rs4)
rs5 = self.ns_count_with_pair(self.ns_leader, name, 'card0', 'card', 'ts2')
self.assertIn('count: 2', rs5)
rs6 = self.ns_count_with_pair(self.ns_leader, name, 'card1', 'card', 'ts1')
self.assertIn('count: 1', rs6)
rs7 = self.ns_count_with_pair(self.ns_leader, name, 'card1', 'card', 'ts2')
self.assertIn('count: 1', rs7)
rs8 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'mcc', 'ts1')
self.assertIn('idx name not found', rs8)
rs9 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'card', 'ts1')
self.assertIn('count: 0', rs9)
rs10 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'k1', 'ts1')
self.assertIn('idx name not found', rs10)
rs11 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'mcc', 'ts3')
self.assertIn('ts name not found', rs11)
def test_two_ts_two_index_latest_ttl(self):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 2,
"ttl_type": "kLatestTime",
"partition_num": 1,
"replica_num": 1,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true", "ttl": 3},
],
"column_key":[
{"index_name":"card", "ts_name":["ts", "ts1"]},
{"index_name":"mcc", "ts_name":["ts"]},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
curtime = int(time.time() * 1000)
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.1', '1', '11'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc1', '1.2', '2', '12'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc2', '1.3', '3', '13'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc3', '1.4', '4', '14'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.6', '5', '15'])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.7', '6', '16'])
self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc1', '1.8', '7', '17'])
self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc1', '1.9', '8', '18'])
self.ns_put_multi(self.ns_leader, name, '', ['card3', 'mcc4', '2.0', '9', '19'])
rs3 = self.makesnapshot(self.ns_leader, name, 0, 'ns_client')
table_info = self.showtable(self.ns_leader, name)
tid = table_info.keys()[0][1]
tablet_endpoint = table_info.keys()[0][3]
self.execute_gc(tablet_endpoint, tid, '0')
rs3 = self.makesnapshot(self.ns_leader, name, '0', 'ns_client')
self.assertIn('MakeSnapshot ok', rs3)
mf = self.get_manifest(self.leaderpath, tid, 0)
self.assertEqual(mf['offset'], '9')
self.assertEqual(mf['count'], '7')
def test_scan_atleast(self):
"""
指定时间列的schema表查询pk下的多条数据
:return:
"""
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 0,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts3", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
],
"column_key":[
{"index_name":"card", "ts_name":["ts1", "ts2", "ts3"]},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
for i in range(100):
rs = self.ns_put_multi_with_pair(self.ns_leader,name, ['card' + str(i%10), str(i+1), str(i+1), str(i+1)])
self.assertIn('Put ok', rs)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '0', 'ts1', '0', '8')
self.assertEqual(len(prs), 10)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '50', 'ts1', '0', '0')
self.assertEqual(len(prs), 5)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '50', 'ts1', '0', '8')
self.assertEqual(len(prs), 8)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '50', '0', 'ts1', '0', '8')
self.assertEqual(len(prs), 5)
for i in range(10):
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '10', 'ts1', '8', '3')
self.assertEqual(len(prs), 8)
prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '90', '0', 'ts1', '4', '5')
self.assertEqual(len(prs), 0)
def test_create_one_ts_table_ttl_desc(self, exp_msg, ttl_type, ttl, *eles):
"""
使用ttl_desc方式创建单ts表，检查ttl类型
"""
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"partition_num": 3,
"replica_num": 3,
"ttl_type": ttl_type,
"ttl_desc": [],
"column_desc": [],
"column_key": []
}
for item in eles:
table_meta[item[0]].append(item[1])
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
infoLogger.info(rs)
self.assertIn(exp_msg, rs)
self.ns_put_multi(self.ns_leader, name,
self.now() - 5 * 60 * 1000, ['card2', '123', '1.1'])
self.ns_put_multi(self.ns_leader, name,
self.now() - 4 * 60 * 1000, ['card0', '123', '1.1'])
self.ns_put_multi(self.ns_leader, name,
self.now() - 3 * 60 * 1000, ['card1', '123', '2.2'])
self.ns_put_multi(self.ns_leader, name,
self.now() - 2 * 60 * 1000, ['card1', '233', '3.3'])
self.ns_put_multi(self.ns_leader, name,
self.now() - 1 * 60 * 1000, ['card0', '233', '4.4'])
self.ns_put_multi(self.ns_leader, name,
self.now() - 0 * 60 * 1000, ['card2', '123', '1.1'])
time.sleep(2)
table_info = self.showtable_with_tablename(self.ns_leader, name)
table_info = self.parse_tb(table_info, ' ', [1, 2, 3], [5, 9])
record_cnt = 0
for k, v in table_info.items():
self.assertEqual(v[0], ttl)
record_cnt += int(v[1])
self.assertEqual(record_cnt, 18)
self.ns_drop(self.ns_leader, name)
def test_two_ts_ttl(self):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name": name,
"ttl": 10,
"partition_num": 1,
"replica_num": 1,
"column_desc":[
{"name": "card", "type": "string", "add_ts_idx": "true"},
{"name": "mcc", "type": "string", "add_ts_idx": "true"},
{"name": "amt", "type": "double", "add_ts_idx": "false"},
{"name": "ts", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"},
{"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true", "ttl": 20},
],
"column_key":[
{"index_name":"card_mcc", "col_name": ["card", "mcc"], "ts_name":["ts", "ts1"]},
{"index_name":"mcc", "ts_name":["ts"]},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
curtime = int(time.time() * 1000)
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.1', str(curtime), str(curtime - 10)])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc1', '1.2', str(curtime - 10), str(curtime)])
self.ns_put_multi(self.ns_leader, name, '', ['card1', 'mcc2', '1.3', str(curtime), str(curtime - 10*60*1000 - 10)])
self.ns_put_multi(self.ns_leader, name, '', ['card1', 'mcc3', '1.4', str(curtime - 10*60*1000 - 10), str(curtime - 30)])
self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc5', '1.6', str(curtime - 10*60*1000 - 10), str(curtime - 20*60*1000 - 20)])
self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc2', '1.7', str(curtime - 20*60*1000 - 20), str(curtime - 10*60*1000 - 10)])
self.ns_put_multi(self.ns_leader, name, '', ['card3', 'mcc6', '1.7', str(curtime - 20*60*1000 - 20), str(curtime - 20*60*1000 - 10)])
rs3 = self.makesnapshot(self.ns_leader, name, 0, 'ns_client')
table_info = self.showtable(self.ns_leader, name)
tid = table_info.keys()[0][1]
rs3 = self.makesnapshot(self.ns_leader, name, '0', 'ns_client')
self.assertIn('MakeSnapshot ok', rs3)
mf = self.get_manifest(self.leaderpath, tid, 0)
self.assertEqual(mf['offset'], '7')
self.assertEqual(mf['count'], '5')
def create_new_table_put(self, data_count, data_thread=1):
self.tname = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
self.tname,
"ttl":
14400,
"replica_num":
3,
"partition_num":
4,
"column_desc": [{
"name": "k1",
"type": "string",
"add_ts_idx": "true"
}, {
"name": "k2",
"type": "string",
"add_ts_idx": "false"
}, {
"name": "k3",
"type": "string",
"add_ts_idx": "false"
}],
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
on = {
'k1': ('string:index', 'testvalue0'),
'k2': ('string', 'testvalue1'),
'k3': ('string', 1.1)
}
values = ["testkey0", "testvalue1", "testvalue2"]
table_info = self.showtable(self.ns_leader, self.tname)
self.tid = int(table_info.keys()[0][1])
self.pid = 0
self.put_large_datas(data_count, 1)
def test_delete_column_key(self):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
14400,
"column_desc": [
{
"name": "card",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "mcc",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "amt",
"type": "double",
"add_ts_idx": "false"
},
{
"name": "col1",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "col2",
"type": "string",
"add_ts_idx": "false"
},
],
"column_key": [{
"index_name": "card_x",
"col_name": "card"
}, {
"index_name": "col",
"col_name": ["col1", "col2"]
}]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 5)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[2], ['2', 'amt', 'double'])
self.assertEqual(len(column_key), 2)
self.assertEqual(column_key[0],
["0", "card_x", "card", "-", "14400min"])
self.assertEqual(column_key[1],
["1", "col", "col1|col2", "-", "14400min"])
row = ['card0', 'mcc0', '1.3', 'col1x', 'col2x']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
row = ['card0', 'mcc1', '1.4', 'col1m', 'col2m']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
row = ['card1', 'mcc1', '1.5', 'col11', 'col22']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card_x',
self.now() + 100, 0)
self.assertEqual(len(rs), 2)
rs5 = self.ns_delete(self.ns_leader, name, 'card0', 'card_x')
self.assertIn('delete ok', rs5)
rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card_x',
self.now() + 100, 0)
self.assertEqual(len(rs), 0)
rs = self.ns_scan_multi(self.ns_leader, name, 'col1m|col2m', 'col',
self.now() + 100, 0)
self.assertEqual(len(rs), 1)
rs5 = self.ns_delete(self.ns_leader, name, 'col1m|col2m', 'col')
self.assertIn('delete ok', rs5)
rs = self.ns_scan_multi(self.ns_leader, name, 'col1m|col2m', 'col',
self.now() + 100, 0)
self.assertEqual(len(rs), 0)
def test_partitionkey_all(self, args):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
0,
"format_version":
args[0],
"column_desc": [
{
"name": "card",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "mcc",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "amt",
"type": "double",
"add_ts_idx": "false"
},
],
"partition_key":
self.convert_partition_key(args[1])
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
row = ['card0', 'mcc0', '1.3']
self.ns_put_multi(self.ns_leader, name, 11, row)
row = ['card0', 'mcc1', '1.4']
self.ns_put_multi(self.ns_leader, name, 22, row)
row = ['card1', 'mcc1', '1.5']
self.ns_put_multi(self.ns_leader, name, 33, row)
rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card', 0, 0)
self.assertEqual(len(rs), 2)
rs = self.ns_scan_multi(self.ns_leader, name, 'mcc0', 'mcc', 0, 0)
self.assertEqual(len(rs), 1)
self.assertEqual(rs[0]['card'], 'card0')
self.assertEqual(rs[0]['mcc'], 'mcc0')
rs = self.ns_scan_multi(self.ns_leader, name, 'mcc1', 'mcc', 0, 0)
self.assertEqual(len(rs), 2)
rs = self.ns_get_multi(self.ns_leader, name, 'mcc1', 'mcc', 0)
self.assertEqual(rs['card'], 'card1')
self.assertEqual(rs['mcc'], 'mcc1')
rs = self.ns_get_multi(self.ns_leader, name, 'card0', 'card', 0)
self.assertEqual(rs['card'], 'card0')
self.assertEqual(rs['mcc'], 'mcc1')
rs = self.ns_count(self.ns_leader, name, 'card0', 'card')
self.assertEqual("count: 2", rs)
rs = self.ns_count(self.ns_leader, name, 'mcc1', 'mcc')
self.assertEqual("count: 2", rs)
rs = self.ns_count(self.ns_leader, name, 'mcc0', 'mcc')
self.assertEqual("count: 1", rs)
rs = self.ns_preview(self.ns_leader, name)
self.assertEqual(len(rs), 3)
rs = self.ns_delete(self.ns_leader, name, "mcc1", "mcc")
rs = self.ns_count(self.ns_leader, name, 'mcc1', 'mcc')
self.assertEqual("count: 0", rs)
self.ns_drop(self.ns_leader, name)
def test_addColAndIndex_multi(self, format_version):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
14400,
"format_version":
format_version,
"partition_num":
8,
"replica_num":
3,
"column_desc": [
{
"name": "card",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "mcc",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "amt",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "num",
"type": "int64",
"add_ts_idx": "false"
},
{
"name": "coll",
"type": "int64",
"add_ts_idx": "false"
},
{
"name": "loc",
"type": "string",
"add_ts_idx": "false"
},
],
"column_key": [
{
"index_name": "card"
},
{
"index_name": "mcc"
},
{
"index_name": "amt"
},
{
"index_name": "num"
},
{
"index_name": "coll"
},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 6)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[4], ['4', 'coll', 'int64'])
self.assertEqual(len(column_key), 5)
self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"])
self.assertEqual(column_key[4], ["4", "coll", "coll", "-", "14400min"])
for i in range(20):
row = [
"card" + str(i), "mcc" + str(i),
str(2.33 + i),
str(i + 123),
str(i + 666), "loc" + str(i)
]
self.ns_put_multi(self.ns_leader, name, self.now(), row)
rs = self.ns_scan_multi(self.ns_leader, name, 'loc0', 'loc',
self.now() + 100, 0)
self.assertEqual(len(rs), 0)
rs = self.ns_addindex(self.ns_leader, name, "add",
"add:int32,add2:string")
self.assertIn('addindex ok', rs)
time.sleep(2)
self.wait_op_done(name)
rs = self.ns_addindex(self.ns_leader, name, "loc")
self.assertIn('addindex ok', rs)
time.sleep(2)
self.wait_op_done(name)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 8)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[4], ['4', 'coll', 'int64'])
self.assertEqual(schema[6], ['6', 'add', 'int32'])
self.assertEqual(schema[7], ['7', 'add2', 'string'])
self.assertEqual(len(column_key), 7)
self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"])
self.assertEqual(column_key[4], ["4", "coll", "coll", "-", "14400min"])
self.assertEqual(column_key[5],
["5", "add", "add|add2", "-", "14400min"])
self.assertEqual(column_key[6], ["6", "loc", "loc", "-", "14400min"])
for i in range(20):
rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i),
'loc',
self.now() + 100, 0)
self.assertEqual(len(rs), 1)
self.assertEqual(rs[0]['card'], 'card' + str(i))
self.assertEqual(rs[0]['mcc'], 'mcc' + str(i))
self.assertEqual(rs[0]['loc'], 'loc' + str(i))
for i in range(20):
row = [
"card" + str(i), "mcc" + str(i),
str(2.33 + i),
str(i + 123),
str(i + 666), "loc{}".format(i), "{}".format(i),
"2add{}".format(i)
]
self.ns_put_multi(self.ns_leader, name, self.now(), row)
for i in range(20):
rs = self.ns_scan_multi(self.ns_leader, name,
'{}|2add{}'.format(i, i), 'add',
self.now() + 100, 0)
self.assertEqual(len(rs), 1)
self.assertEqual(rs[0]['card'], 'card' + str(i))
self.assertEqual(rs[0]['mcc'], 'mcc' + str(i))
self.assertEqual(rs[0]['add'], str(i))
for i in range(20):
rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i),
'loc',
self.now() + 100, 0)
self.assertEqual(len(rs), 2)
self.assertEqual(rs[0]['card'], 'card' + str(i))
self.assertEqual(rs[0]['mcc'], 'mcc' + str(i))
self.assertEqual(rs[0]['loc'], 'loc' + str(i))
self.assertEqual(rs[0]['add2'], "2add{}".format(i))
rs = self.ns_deleteindex(self.ns_leader, name, "loc")
self.assertIn('delete index ok', rs)
time.sleep(2)
rs = self.ns_addindex(self.ns_leader, name, "loc2", "loc")
self.assertIn('addindex ok', rs)
time.sleep(2)
self.wait_op_done(name)
time.sleep(2)
for i in range(20):
rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i),
'loc2', 0, 0)
self.assertEqual(len(rs), 2)
self.assertEqual(rs[0]['card'], 'card' + str(i))
self.assertEqual(rs[0]['mcc'], 'mcc' + str(i))
self.assertEqual(rs[0]['loc'], 'loc' + str(i))
self.assertEqual(rs[0]['add2'], "2add{}".format(i))
self.ns_drop(self.ns_leader, name)
def test_addindex(self, format_version):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
14400,
"format_version":
format_version,
"column_desc": [
{
"name": "card",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "mcc",
"type": "string",
"add_ts_idx": "false"
},
{
"name": "amt",
"type": "double",
"add_ts_idx": "false"
},
],
"column_key": [
{
"index_name": "card"
},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 3)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[2], ['2', 'amt', 'double'])
self.assertEqual(len(column_key), 1)
self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"])
row = ['card0', 'mcc0', '1.3']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
row = ['card0', 'mcc1', '1.4']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
row = ['card1', 'mcc1', '1.5']
self.ns_put_multi(self.ns_leader, name, self.now(), row)
rs = self.ns_scan_multi(self.ns_leader, name, 'mcc0', 'mcc',
self.now() + 100, 0)
self.assertEqual(len(rs), 0)
rs = self.ns_addindex(self.ns_leader, name, "mcc")
self.assertIn('addindex ok', rs)
time.sleep(2)
self.wait_op_done(name)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(schema), 3)
self.assertEqual(schema[0], ['0', 'card', 'string'])
self.assertEqual(schema[2], ['2', 'amt', 'double'])
self.assertEqual(len(column_key), 2)
self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"])
self.assertEqual(column_key[1], ["1", "mcc", "mcc", "-", "14400min"])
rs = self.ns_scan_multi(self.ns_leader, name, 'mcc0', 'mcc',
self.now() + 100, 0)
self.assertEqual(len(rs), 1)
self.assertEqual(rs[0]['card'], 'card0')
self.assertEqual(rs[0]['mcc'], 'mcc0')
self.ns_drop(self.ns_leader, name)
def test_set_ttl_ts_name_latest(self):
name = 'tname{}'.format(time.time())
metadata_path = '{}/metadata.txt'.format(self.testpath)
table_meta = {
"name":
name,
"ttl":
100,
"ttl_type":
"kLatestTime",
"column_desc": [
{
"name": "card",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "mcc",
"type": "string",
"add_ts_idx": "true"
},
{
"name": "amt",
"type": "double",
"add_ts_idx": "false"
},
{
"name": "ts1",
"type": "int64",
"add_ts_idx": "false",
"is_ts_col": "true",
"ttl": 10
},
{
"name": "ts2",
"type": "int64",
"add_ts_idx": "false",
"is_ts_col": "true"
},
],
"column_key": [
{
"index_name": "card",
"ts_name": ["ts1", "ts2"]
},
{
"index_name": "mcc",
"ts_name": ["ts2"]
},
]
}
utils.gen_table_meta_file(table_meta, metadata_path)
rs = self.ns_create(self.ns_leader, metadata_path)
self.assertIn('Create table ok', rs)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(column_key), 3)
self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "10"])
self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "100"])
self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "100"])
rs1 = self.ns_setttl(self.ns_leader, 'setttl', name, 'latest', 1,
'ts2')
self.assertIn('Set ttl ok !', rs1)
(schema, column_key) = self.ns_showschema(self.ns_leader, name)
self.assertEqual(len(column_key), 3)
self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "10"])
self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "1"])
self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "1"])
self.ns_drop(self.ns_leader, name)
