def test_add_table_field_without_columnkey(self): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 14400, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "mcc", "type": "string", "add_ts_idx": "true"}, {"name": "amt", "type": "double", "add_ts_idx": "false"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false"} ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 4) self.assertEqual(schema[0], ["0", "card", "string", "yes"]) self.assertEqual(schema[1], ["1", "mcc", "string", "yes"]) self.assertEqual(schema[2], ["2", "amt", "double", "no"]) self.assertEqual(schema[3], ["3", "ts1", "int64", "no"]) self.ns_add_table_field(self.ns_leader, name, 'aa', 'string'); (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 5) self.assertEqual(schema[0], ["0", "card", "string", "yes"]) self.assertEqual(schema[1], ["1", "mcc", "string", "yes"]) self.assertEqual(schema[2], ["2", "amt", "double", "no"]) self.assertEqual(schema[3], ["3", "ts1", "int64", "no"]) self.assertEqual(schema[4], ["4", "aa", "string", "no"]) self.ns_drop(self.ns_leader, name)
def test_create_showtable_record_cnt(self, exp_msg, *eles): """ 使用文件column_key创建表,put数据后showtable检查record的数量 """ name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 120, "partition_num": 3, "replica_num": 3, "column_desc": [], "column_key": [] } for item in eles: table_meta[item[0]].append(item[1]) utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) infoLogger.info(rs) self.assertIn(exp_msg, rs) self.ns_put_multi(self.ns_leader, name, 10, ['card0', '123', '1.1']) self.ns_put_multi(self.ns_leader, name, 10, ['card1', '123', '2.2']) self.ns_put_multi(self.ns_leader, name, 10, ['card1', '233', '3.3']) self.ns_put_multi(self.ns_leader, name, 10, ['card0', '233', '4.4']) time.sleep(2) table_info = self.showtable_with_tablename(self.ns_leader, name) table_info = self.parse_tb(table_info, ' ', [1, 2, 3], [9]) record_cnt = 0 for k, v in table_info.items(): record_cnt += int(v[0]) self.assertEqual(record_cnt, 12) self.ns_drop(self.ns_leader, name)
def test_partitionkey_create(self, args): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 0, "column_desc": [ { "name": "col1", "type": "string", "add_ts_idx": "true" }, { "name": "col2", "type": "string", "add_ts_idx": "false" }, { "name": "col3", "type": "double", "add_ts_idx": "false" }, ], "partition_key": self.convert_partition_key(args[0]) } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn(args[1], rs) if args[1] == "Create table ok": result = self.ns_info(self.ns_leader, name) self.assertEqual(result["partition_key"].replace(" ", ""), args[0]) self.ns_drop(self.ns_leader, name)
def test_showschema_no_columnkey(self): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 14400, "column_desc": [ { "name": "card", "type": "string", "add_ts_idx": "true" }, { "name": "mcc", "type": "string", "add_ts_idx": "true" }, { "name": "amt", "type": "double", "add_ts_idx": "false" }, { "name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true" }, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 4) self.assertEqual(schema[0], ["0", "card", "string"]) self.assertEqual(schema[2], ["2", "amt", "double"]) self.assertEqual(schema[3], ["3", "ts1", "int64"]) self.assertEqual(len(column_key), 2) self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "14400min"]) self.assertEqual(column_key[1], ["1", "mcc", "mcc", "ts1", "14400min"]) rs = self.ns_deleteindex(self.ns_leader, name, "card") self.assertIn('Fail to delete index', rs) rs = self.ns_deleteindex(self.ns_leader, name, "mcc") self.assertIn('delete index ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 4) self.assertEqual(schema[0], ["0", "card", "string"]) self.assertEqual(schema[2], ["2", "amt", "double"]) self.assertEqual(schema[3], ["3", "ts1", "int64"]) self.assertEqual(len(column_key), 1) self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "14400min"]) self.ns_drop(self.ns_leader, name)
def test_scan_schema_has_ts_col(self): """ 指定时间列的schema表查询pk下的多条数据 :return: """ name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 0, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "mcc", "type": "string", "add_ts_idx": "true"}, {"name": "amt", "type": "double", "add_ts_idx": "false"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, ], "column_key":[ {"index_name":"card", "ts_name":["ts1", "ts2"]}, {"index_name":"mcc", "ts_name":["ts2"]}, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) rs1 = self.ns_put_multi_with_pair(self.ns_leader,name, ['card0', 'mcc0', '1', '15', '18']) self.assertIn('Put ok', rs1) rs2 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card0', 'mcc1', '2', '16', '19']) self.assertIn('Put ok', rs2) rs3 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card1', 'mcc2', '3', '20', '20']) self.assertIn('Put ok', rs3) rs4 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card0', 'card', '25', '0', 'ts1') self.assertEqual(len(rs4),2) self.assertEqual(rs4[0]['card'], 'card0') self.assertEqual(rs4[0]['mcc'], 'mcc1') self.assertEqual(rs4[0]['amt'], '2') self.assertEqual(rs4[0]['ts1'], '16') self.assertEqual(rs4[0]['ts2'], '19') self.assertEqual(rs4[1]['card'], 'card0') self.assertEqual(rs4[1]['mcc'], 'mcc0') self.assertEqual(rs4[1]['amt'], '1') self.assertEqual(rs4[1]['ts1'], '15') self.assertEqual(rs4[1]['ts2'], '18') rs5 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card0', 'card', '25', '0', 'ts1', '1') self.assertEqual(len(rs5),1) self.assertEqual(rs5[0]['card'], 'card0') self.assertEqual(rs5[0]['mcc'], 'mcc1') self.assertEqual(rs5[0]['amt'], '2') self.assertEqual(rs5[0]['ts1'], '16') self.assertEqual(rs5[0]['ts2'], '19') rs6 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'mcc1', 'mcc', '25', '0', 'ts1', '1') self.assertEqual(len(rs6), 0) rs7 = self.ns_scan_multi_with_pair(self.ns_leader, name, 'mcc1', 'mcc', '25', '0', 'ts2', '1') self.assertEqual(len(rs7), 1)
def test_add_table_field_with_columnkey(self): """ 指定时间列的schema表统计pk下的条数 :return: """ name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 0, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "mcc", "type": "string", "add_ts_idx": "true"}, {"name": "amt", "type": "double", "add_ts_idx": "false"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, ], "column_key":[ {"index_name":"card", "ts_name":["ts1", "ts2"]}, {"index_name":"mcc", "ts_name":["ts2"]}, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 5) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[1], ['1', 'mcc', 'string']) self.assertEqual(schema[2], ['2', 'amt', 'double']) self.assertEqual(schema[3], ['3', 'ts1', 'int64']) self.assertEqual(schema[4], ['4', 'ts2', 'int64']) self.assertEqual(len(column_key), 3) self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "0min"]) self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "0min"]) self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "0min"]) self.ns_add_table_field(self.ns_leader, name, 'aa', 'string'); (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 6) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[1], ['1', 'mcc', 'string']) self.assertEqual(schema[2], ['2', 'amt', 'double']) self.assertEqual(schema[3], ['3', 'ts1', 'int64']) self.assertEqual(schema[4], ['4', 'ts2', 'int64']) self.assertEqual(schema[5], ['5', 'aa', 'string']) self.assertEqual(len(column_key), 3) self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "0min"]) self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "0min"]) self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "0min"]) self.ns_drop(self.ns_leader, name)
def test_count_schema_has_ts_col(self): """ 指定时间列的schema表统计pk下的条数 :return: """ name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 0, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "mcc", "type": "string", "add_ts_idx": "true"}, {"name": "amt", "type": "double", "add_ts_idx": "false"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, ], "column_key":[ {"index_name":"card", "ts_name":["ts1", "ts2"]}, {"index_name":"mcc", "ts_name":["ts2"]}, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) rs1 = self.ns_put_multi_with_pair(self.ns_leader,name, ['card0', 'mcc0', '1', '15', '18']) self.assertIn('Put ok', rs1) rs2 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card0', 'mcc1', '2', '16', '19']) self.assertIn('Put ok', rs2) rs3 = self.ns_put_multi_with_pair(self.ns_leader, name, ['card1', 'mcc2', '3', '20', '20']) self.assertIn('Put ok', rs3) rs4 = self.ns_count_with_pair(self.ns_leader, name, 'card0', 'card', 'ts1') self.assertIn('count: 2', rs4) rs5 = self.ns_count_with_pair(self.ns_leader, name, 'card0', 'card', 'ts2') self.assertIn('count: 2', rs5) rs6 = self.ns_count_with_pair(self.ns_leader, name, 'card1', 'card', 'ts1') self.assertIn('count: 1', rs6) rs7 = self.ns_count_with_pair(self.ns_leader, name, 'card1', 'card', 'ts2') self.assertIn('count: 1', rs7) rs8 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'mcc', 'ts1') self.assertIn('idx name not found', rs8) rs9 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'card', 'ts1') self.assertIn('count: 0', rs9) rs10 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'k1', 'ts1') self.assertIn('idx name not found', rs10) rs11 = self.ns_count_with_pair(self.ns_leader, name, 'mcc1', 'mcc', 'ts3') self.assertIn('ts name not found', rs11)
def test_two_ts_two_index_latest_ttl(self): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 2, "ttl_type": "kLatestTime", "partition_num": 1, "replica_num": 1, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "mcc", "type": "string", "add_ts_idx": "true"}, {"name": "amt", "type": "double", "add_ts_idx": "false"}, {"name": "ts", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true", "ttl": 3}, ], "column_key":[ {"index_name":"card", "ts_name":["ts", "ts1"]}, {"index_name":"mcc", "ts_name":["ts"]}, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) curtime = int(time.time() * 1000) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.1', '1', '11']) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc1', '1.2', '2', '12']) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc2', '1.3', '3', '13']) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc3', '1.4', '4', '14']) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.6', '5', '15']) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.7', '6', '16']) self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc1', '1.8', '7', '17']) self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc1', '1.9', '8', '18']) self.ns_put_multi(self.ns_leader, name, '', ['card3', 'mcc4', '2.0', '9', '19']) rs3 = self.makesnapshot(self.ns_leader, name, 0, 'ns_client') table_info = self.showtable(self.ns_leader, name) tid = table_info.keys()[0][1] tablet_endpoint = table_info.keys()[0][3] self.execute_gc(tablet_endpoint, tid, '0') rs3 = self.makesnapshot(self.ns_leader, name, '0', 'ns_client') self.assertIn('MakeSnapshot ok', rs3) mf = self.get_manifest(self.leaderpath, tid, 0) self.assertEqual(mf['offset'], '9') self.assertEqual(mf['count'], '7')
def test_scan_atleast(self): """ 指定时间列的schema表查询pk下的多条数据 :return: """ name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 0, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts3", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, ], "column_key":[ {"index_name":"card", "ts_name":["ts1", "ts2", "ts3"]}, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) for i in range(100): rs = self.ns_put_multi_with_pair(self.ns_leader,name, ['card' + str(i%10), str(i+1), str(i+1), str(i+1)]) self.assertIn('Put ok', rs) for i in range(10): prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '0', 'ts1', '0', '8') self.assertEqual(len(prs), 10) for i in range(10): prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '50', 'ts1', '0', '0') self.assertEqual(len(prs), 5) for i in range(10): prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '50', 'ts1', '0', '8') self.assertEqual(len(prs), 8) for i in range(10): prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '50', '0', 'ts1', '0', '8') self.assertEqual(len(prs), 5) for i in range(10): prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '100', '10', 'ts1', '8', '3') self.assertEqual(len(prs), 8) prs = self.ns_scan_multi_with_pair(self.ns_leader, name, 'card'+str(i), 'card', '90', '0', 'ts1', '4', '5') self.assertEqual(len(prs), 0)
def test_create_one_ts_table_ttl_desc(self, exp_msg, ttl_type, ttl, *eles): """ 使用ttl_desc方式创建单ts表,检查ttl类型 """ name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "partition_num": 3, "replica_num": 3, "ttl_type": ttl_type, "ttl_desc": [], "column_desc": [], "column_key": [] } for item in eles: table_meta[item[0]].append(item[1]) utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) infoLogger.info(rs) self.assertIn(exp_msg, rs) self.ns_put_multi(self.ns_leader, name, self.now() - 5 * 60 * 1000, ['card2', '123', '1.1']) self.ns_put_multi(self.ns_leader, name, self.now() - 4 * 60 * 1000, ['card0', '123', '1.1']) self.ns_put_multi(self.ns_leader, name, self.now() - 3 * 60 * 1000, ['card1', '123', '2.2']) self.ns_put_multi(self.ns_leader, name, self.now() - 2 * 60 * 1000, ['card1', '233', '3.3']) self.ns_put_multi(self.ns_leader, name, self.now() - 1 * 60 * 1000, ['card0', '233', '4.4']) self.ns_put_multi(self.ns_leader, name, self.now() - 0 * 60 * 1000, ['card2', '123', '1.1']) time.sleep(2) table_info = self.showtable_with_tablename(self.ns_leader, name) table_info = self.parse_tb(table_info, ' ', [1, 2, 3], [5, 9]) record_cnt = 0 for k, v in table_info.items(): self.assertEqual(v[0], ttl) record_cnt += int(v[1]) self.assertEqual(record_cnt, 18) self.ns_drop(self.ns_leader, name)
def test_two_ts_ttl(self): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 10, "partition_num": 1, "replica_num": 1, "column_desc":[ {"name": "card", "type": "string", "add_ts_idx": "true"}, {"name": "mcc", "type": "string", "add_ts_idx": "true"}, {"name": "amt", "type": "double", "add_ts_idx": "false"}, {"name": "ts", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true"}, {"name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true", "ttl": 20}, ], "column_key":[ {"index_name":"card_mcc", "col_name": ["card", "mcc"], "ts_name":["ts", "ts1"]}, {"index_name":"mcc", "ts_name":["ts"]}, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) curtime = int(time.time() * 1000) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc0', '1.1', str(curtime), str(curtime - 10)]) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc1', '1.2', str(curtime - 10), str(curtime)]) self.ns_put_multi(self.ns_leader, name, '', ['card1', 'mcc2', '1.3', str(curtime), str(curtime - 10*60*1000 - 10)]) self.ns_put_multi(self.ns_leader, name, '', ['card1', 'mcc3', '1.4', str(curtime - 10*60*1000 - 10), str(curtime - 30)]) self.ns_put_multi(self.ns_leader, name, '', ['card2', 'mcc5', '1.6', str(curtime - 10*60*1000 - 10), str(curtime - 20*60*1000 - 20)]) self.ns_put_multi(self.ns_leader, name, '', ['card0', 'mcc2', '1.7', str(curtime - 20*60*1000 - 20), str(curtime - 10*60*1000 - 10)]) self.ns_put_multi(self.ns_leader, name, '', ['card3', 'mcc6', '1.7', str(curtime - 20*60*1000 - 20), str(curtime - 20*60*1000 - 10)]) rs3 = self.makesnapshot(self.ns_leader, name, 0, 'ns_client') table_info = self.showtable(self.ns_leader, name) tid = table_info.keys()[0][1] rs3 = self.makesnapshot(self.ns_leader, name, '0', 'ns_client') self.assertIn('MakeSnapshot ok', rs3) mf = self.get_manifest(self.leaderpath, tid, 0) self.assertEqual(mf['offset'], '7') self.assertEqual(mf['count'], '5')
def create_new_table_put(self, data_count, data_thread=1): self.tname = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": self.tname, "ttl": 14400, "replica_num": 3, "partition_num": 4, "column_desc": [{ "name": "k1", "type": "string", "add_ts_idx": "true" }, { "name": "k2", "type": "string", "add_ts_idx": "false" }, { "name": "k3", "type": "string", "add_ts_idx": "false" }], } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) on = { 'k1': ('string:index', 'testvalue0'), 'k2': ('string', 'testvalue1'), 'k3': ('string', 1.1) } values = ["testkey0", "testvalue1", "testvalue2"] table_info = self.showtable(self.ns_leader, self.tname) self.tid = int(table_info.keys()[0][1]) self.pid = 0 self.put_large_datas(data_count, 1)
def test_delete_column_key(self): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 14400, "column_desc": [ { "name": "card", "type": "string", "add_ts_idx": "false" }, { "name": "mcc", "type": "string", "add_ts_idx": "false" }, { "name": "amt", "type": "double", "add_ts_idx": "false" }, { "name": "col1", "type": "string", "add_ts_idx": "false" }, { "name": "col2", "type": "string", "add_ts_idx": "false" }, ], "column_key": [{ "index_name": "card_x", "col_name": "card" }, { "index_name": "col", "col_name": ["col1", "col2"] }] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 5) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[2], ['2', 'amt', 'double']) self.assertEqual(len(column_key), 2) self.assertEqual(column_key[0], ["0", "card_x", "card", "-", "14400min"]) self.assertEqual(column_key[1], ["1", "col", "col1|col2", "-", "14400min"]) row = ['card0', 'mcc0', '1.3', 'col1x', 'col2x'] self.ns_put_multi(self.ns_leader, name, self.now(), row) row = ['card0', 'mcc1', '1.4', 'col1m', 'col2m'] self.ns_put_multi(self.ns_leader, name, self.now(), row) row = ['card1', 'mcc1', '1.5', 'col11', 'col22'] self.ns_put_multi(self.ns_leader, name, self.now(), row) rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card_x', self.now() + 100, 0) self.assertEqual(len(rs), 2) rs5 = self.ns_delete(self.ns_leader, name, 'card0', 'card_x') self.assertIn('delete ok', rs5) rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card_x', self.now() + 100, 0) self.assertEqual(len(rs), 0) rs = self.ns_scan_multi(self.ns_leader, name, 'col1m|col2m', 'col', self.now() + 100, 0) self.assertEqual(len(rs), 1) rs5 = self.ns_delete(self.ns_leader, name, 'col1m|col2m', 'col') self.assertIn('delete ok', rs5) rs = self.ns_scan_multi(self.ns_leader, name, 'col1m|col2m', 'col', self.now() + 100, 0) self.assertEqual(len(rs), 0)
def test_partitionkey_all(self, args): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 0, "format_version": args[0], "column_desc": [ { "name": "card", "type": "string", "add_ts_idx": "true" }, { "name": "mcc", "type": "string", "add_ts_idx": "true" }, { "name": "amt", "type": "double", "add_ts_idx": "false" }, ], "partition_key": self.convert_partition_key(args[1]) } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) row = ['card0', 'mcc0', '1.3'] self.ns_put_multi(self.ns_leader, name, 11, row) row = ['card0', 'mcc1', '1.4'] self.ns_put_multi(self.ns_leader, name, 22, row) row = ['card1', 'mcc1', '1.5'] self.ns_put_multi(self.ns_leader, name, 33, row) rs = self.ns_scan_multi(self.ns_leader, name, 'card0', 'card', 0, 0) self.assertEqual(len(rs), 2) rs = self.ns_scan_multi(self.ns_leader, name, 'mcc0', 'mcc', 0, 0) self.assertEqual(len(rs), 1) self.assertEqual(rs[0]['card'], 'card0') self.assertEqual(rs[0]['mcc'], 'mcc0') rs = self.ns_scan_multi(self.ns_leader, name, 'mcc1', 'mcc', 0, 0) self.assertEqual(len(rs), 2) rs = self.ns_get_multi(self.ns_leader, name, 'mcc1', 'mcc', 0) self.assertEqual(rs['card'], 'card1') self.assertEqual(rs['mcc'], 'mcc1') rs = self.ns_get_multi(self.ns_leader, name, 'card0', 'card', 0) self.assertEqual(rs['card'], 'card0') self.assertEqual(rs['mcc'], 'mcc1') rs = self.ns_count(self.ns_leader, name, 'card0', 'card') self.assertEqual("count: 2", rs) rs = self.ns_count(self.ns_leader, name, 'mcc1', 'mcc') self.assertEqual("count: 2", rs) rs = self.ns_count(self.ns_leader, name, 'mcc0', 'mcc') self.assertEqual("count: 1", rs) rs = self.ns_preview(self.ns_leader, name) self.assertEqual(len(rs), 3) rs = self.ns_delete(self.ns_leader, name, "mcc1", "mcc") rs = self.ns_count(self.ns_leader, name, 'mcc1', 'mcc') self.assertEqual("count: 0", rs) self.ns_drop(self.ns_leader, name)
def test_addColAndIndex_multi(self, format_version): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 14400, "format_version": format_version, "partition_num": 8, "replica_num": 3, "column_desc": [ { "name": "card", "type": "string", "add_ts_idx": "false" }, { "name": "mcc", "type": "string", "add_ts_idx": "false" }, { "name": "amt", "type": "string", "add_ts_idx": "false" }, { "name": "num", "type": "int64", "add_ts_idx": "false" }, { "name": "coll", "type": "int64", "add_ts_idx": "false" }, { "name": "loc", "type": "string", "add_ts_idx": "false" }, ], "column_key": [ { "index_name": "card" }, { "index_name": "mcc" }, { "index_name": "amt" }, { "index_name": "num" }, { "index_name": "coll" }, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 6) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[4], ['4', 'coll', 'int64']) self.assertEqual(len(column_key), 5) self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"]) self.assertEqual(column_key[4], ["4", "coll", "coll", "-", "14400min"]) for i in range(20): row = [ "card" + str(i), "mcc" + str(i), str(2.33 + i), str(i + 123), str(i + 666), "loc" + str(i) ] self.ns_put_multi(self.ns_leader, name, self.now(), row) rs = self.ns_scan_multi(self.ns_leader, name, 'loc0', 'loc', self.now() + 100, 0) self.assertEqual(len(rs), 0) rs = self.ns_addindex(self.ns_leader, name, "add", "add:int32,add2:string") self.assertIn('addindex ok', rs) time.sleep(2) self.wait_op_done(name) rs = self.ns_addindex(self.ns_leader, name, "loc") self.assertIn('addindex ok', rs) time.sleep(2) self.wait_op_done(name) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 8) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[4], ['4', 'coll', 'int64']) self.assertEqual(schema[6], ['6', 'add', 'int32']) self.assertEqual(schema[7], ['7', 'add2', 'string']) self.assertEqual(len(column_key), 7) self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"]) self.assertEqual(column_key[4], ["4", "coll", "coll", "-", "14400min"]) self.assertEqual(column_key[5], ["5", "add", "add|add2", "-", "14400min"]) self.assertEqual(column_key[6], ["6", "loc", "loc", "-", "14400min"]) for i in range(20): rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i), 'loc', self.now() + 100, 0) self.assertEqual(len(rs), 1) self.assertEqual(rs[0]['card'], 'card' + str(i)) self.assertEqual(rs[0]['mcc'], 'mcc' + str(i)) self.assertEqual(rs[0]['loc'], 'loc' + str(i)) for i in range(20): row = [ "card" + str(i), "mcc" + str(i), str(2.33 + i), str(i + 123), str(i + 666), "loc{}".format(i), "{}".format(i), "2add{}".format(i) ] self.ns_put_multi(self.ns_leader, name, self.now(), row) for i in range(20): rs = self.ns_scan_multi(self.ns_leader, name, '{}|2add{}'.format(i, i), 'add', self.now() + 100, 0) self.assertEqual(len(rs), 1) self.assertEqual(rs[0]['card'], 'card' + str(i)) self.assertEqual(rs[0]['mcc'], 'mcc' + str(i)) self.assertEqual(rs[0]['add'], str(i)) for i in range(20): rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i), 'loc', self.now() + 100, 0) self.assertEqual(len(rs), 2) self.assertEqual(rs[0]['card'], 'card' + str(i)) self.assertEqual(rs[0]['mcc'], 'mcc' + str(i)) self.assertEqual(rs[0]['loc'], 'loc' + str(i)) self.assertEqual(rs[0]['add2'], "2add{}".format(i)) rs = self.ns_deleteindex(self.ns_leader, name, "loc") self.assertIn('delete index ok', rs) time.sleep(2) rs = self.ns_addindex(self.ns_leader, name, "loc2", "loc") self.assertIn('addindex ok', rs) time.sleep(2) self.wait_op_done(name) time.sleep(2) for i in range(20): rs = self.ns_scan_multi(self.ns_leader, name, 'loc' + str(i), 'loc2', 0, 0) self.assertEqual(len(rs), 2) self.assertEqual(rs[0]['card'], 'card' + str(i)) self.assertEqual(rs[0]['mcc'], 'mcc' + str(i)) self.assertEqual(rs[0]['loc'], 'loc' + str(i)) self.assertEqual(rs[0]['add2'], "2add{}".format(i)) self.ns_drop(self.ns_leader, name)
def test_addindex(self, format_version): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 14400, "format_version": format_version, "column_desc": [ { "name": "card", "type": "string", "add_ts_idx": "false" }, { "name": "mcc", "type": "string", "add_ts_idx": "false" }, { "name": "amt", "type": "double", "add_ts_idx": "false" }, ], "column_key": [ { "index_name": "card" }, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 3) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[2], ['2', 'amt', 'double']) self.assertEqual(len(column_key), 1) self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"]) row = ['card0', 'mcc0', '1.3'] self.ns_put_multi(self.ns_leader, name, self.now(), row) row = ['card0', 'mcc1', '1.4'] self.ns_put_multi(self.ns_leader, name, self.now(), row) row = ['card1', 'mcc1', '1.5'] self.ns_put_multi(self.ns_leader, name, self.now(), row) rs = self.ns_scan_multi(self.ns_leader, name, 'mcc0', 'mcc', self.now() + 100, 0) self.assertEqual(len(rs), 0) rs = self.ns_addindex(self.ns_leader, name, "mcc") self.assertIn('addindex ok', rs) time.sleep(2) self.wait_op_done(name) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(schema), 3) self.assertEqual(schema[0], ['0', 'card', 'string']) self.assertEqual(schema[2], ['2', 'amt', 'double']) self.assertEqual(len(column_key), 2) self.assertEqual(column_key[0], ["0", "card", "card", "-", "14400min"]) self.assertEqual(column_key[1], ["1", "mcc", "mcc", "-", "14400min"]) rs = self.ns_scan_multi(self.ns_leader, name, 'mcc0', 'mcc', self.now() + 100, 0) self.assertEqual(len(rs), 1) self.assertEqual(rs[0]['card'], 'card0') self.assertEqual(rs[0]['mcc'], 'mcc0') self.ns_drop(self.ns_leader, name)
def test_set_ttl_ts_name_latest(self): name = 'tname{}'.format(time.time()) metadata_path = '{}/metadata.txt'.format(self.testpath) table_meta = { "name": name, "ttl": 100, "ttl_type": "kLatestTime", "column_desc": [ { "name": "card", "type": "string", "add_ts_idx": "true" }, { "name": "mcc", "type": "string", "add_ts_idx": "true" }, { "name": "amt", "type": "double", "add_ts_idx": "false" }, { "name": "ts1", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true", "ttl": 10 }, { "name": "ts2", "type": "int64", "add_ts_idx": "false", "is_ts_col": "true" }, ], "column_key": [ { "index_name": "card", "ts_name": ["ts1", "ts2"] }, { "index_name": "mcc", "ts_name": ["ts2"] }, ] } utils.gen_table_meta_file(table_meta, metadata_path) rs = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(column_key), 3) self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "10"]) self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "100"]) self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "100"]) rs1 = self.ns_setttl(self.ns_leader, 'setttl', name, 'latest', 1, 'ts2') self.assertIn('Set ttl ok !', rs1) (schema, column_key) = self.ns_showschema(self.ns_leader, name) self.assertEqual(len(column_key), 3) self.assertEqual(column_key[0], ["0", "card", "card", "ts1", "10"]) self.assertEqual(column_key[1], ["1", "card", "card", "ts2", "1"]) self.assertEqual(column_key[2], ["2", "mcc", "mcc", "ts2", "1"]) self.ns_drop(self.ns_leader, name)