def parse_tb(rs, splitor, key_cols_index, value_cols_index): """ parse table format response msg :param rs: :param splitor: :param key_cols_index: :param value_cols_index: :return: """ table_dict = {} rs_tb = rs.split('\n') real_conent_flag = False for line in rs_tb: if '------' in line: real_conent_flag = True continue if real_conent_flag: elements = line.split(splitor) elements = [x for x in elements if x != ''] try: k = [elements[x] for x in key_cols_index] v = [elements[x] for x in value_cols_index] if len(key_cols_index) <= 1: key = k[0] else: key = tuple(k) table_dict[key] = v except Exception as e: traceback.print_exc(file=sys.stdout) infoLogger.error(e) if real_conent_flag is False: return rs return table_dict
def fn(arg_self): for data in data_list: if isinstance(data, list): try: f(arg_self, *data) except Exception, e: infoLogger.error('{}({}) failed: {}'.format( f.__name__, data, e)) print '{}({}) failed: {}'.format(f.__name__, data, e) infoLogger.info(dir(f)) else: infoLogger.error('data_list type must be a list')
def get_table_status(self, endpoint, tid='', pid=''): try: rs = self.run_client(endpoint, 'gettablestatus {} {}'.format(tid, pid)) tablestatus = self.parse_tb(rs, ' ', [0, 1], [2, 3, 4, 5, 6, 7, 8]) tableststus_d = {(int(k[0]), int(k[1])): v for k, v in tablestatus.items()} if tid != '': return tableststus_d[(int(tid), int(pid))] else: return tableststus_d except KeyError, e: traceback.print_exc(file=sys.stdout) infoLogger.error('table {} is not exist!'.format(e))
def test_changeleader_master_disconnect(self): """ changeleader功能正常,主节点断网后,可以手工故障切换,切换成功后从节点可以同步数据 :return: """ self.start_client(self.leader) metadata_path = '{}/metadata.txt'.format(self.testpath) name = 'tname{}'.format(time.time()) infoLogger.info(name) m = utils.gen_table_metadata( '"{}"'.format(name), None, 144000, 2, ('table_partition', '"{}"'.format(self.leader), '"0-2"', 'true'), ('table_partition', '"{}"'.format(self.slave1), '"0-1"', 'false'), ('table_partition', '"{}"'.format(self.slave2), '"0-2"', 'false'), ('column_desc', '"k1"', '"string"', 'true'), ('column_desc', '"k2"', '"string"', 'false'), ('column_desc', '"k3"', '"string"', 'false') ) utils.gen_table_metadata_file(m, metadata_path) rs0 = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs0) self.multidimension_vk = {'k1': ('string:index', 'testvalue0'), 'k2': ('string', 'testvalue0'), 'k3': ('string', 'testvalue0')} self.multidimension_scan_vk = {'k1': 'testvalue0'} rs1 = self.showtable(self.ns_leader, name) tid = rs1.keys()[0][1] rs = self.disconnectzk(self.leader) self.assertIn('disconnect zk ok', rs) rs = self.updatetablealive(self.ns_leader, name, '*', self.leader, 'no') self.assertIn('update ok', rs) time.sleep(3) rs = self.changeleader(self.ns_leader, name, 0, 'auto') self.assertIn('change leader ok', rs) time.sleep(2) rs = self.connectzk(self.leader) self.assertIn('connect zk ok', rs) for repeat in range(20): time.sleep(2) rs = self.ns_showopstatus(self.ns_leader) tablestatus = self.parse_tb(rs, ' ', [0, 1, 2, 3], [4, 5, 6, 7]) kDone_cnt = 0 cnt = 0 flag = False for status in tablestatus: if status[2] == name: cnt = cnt + 1 infoLogger.info('{} = {}'.format(status, tablestatus[status])) if tablestatus[status][0] == 'kFailed': infoLogger.error('{} = {}'.format(status, tablestatus[status])) flag = True break if tablestatus[status][0] == 'kDone': infoLogger.info('{} = {}'.format(status, tablestatus[status])) kDone_cnt = kDone_cnt + 1 if flag == True: break if kDone_cnt == cnt: break rs2 = self.showtable(self.ns_leader) act1 = rs2[(name, tid, '0', self.slave1)] act2 = rs2[(name, tid, '0', self.slave2)] roles = [x[0] for x in [act1, act2]] for repeat in range(20): rs = self.showtable(self.ns_leader, name) act1 = rs2[(name, tid, '0', self.slave1)] act2 = rs2[(name, tid, '0', self.slave2)] roles = [x[0] for x in [act1, act2]] if roles.count('leader') == 1 and roles.count('follower') == 1: self.assertEqual(roles.count('leader'), 1) self.assertEqual(roles.count('follower'), 1) break time.sleep(2) self.assertEqual(rs2[(name, tid, '0', self.leader)], ['leader', '144000min', 'no', 'kNoCompress']) self.assertEqual(rs2[(name, tid, '1', self.leader)], ['leader', '144000min', 'no', 'kNoCompress']) self.assertEqual(rs2[(name, tid, '2', self.leader)], ['leader', '144000min', 'no', 'kNoCompress']) if roles.count('leader') != 1: rs = self.ns_showopstatus(self.ns_leader) infoLogger.debug(rs) self.assertEqual(roles.count('leader'), 1) self.assertEqual(roles.count('leader'), 1) self.assertEqual(roles.count('follower'), 1) leader_new = self.slave1 if 'leader' in act1 else self.slave2 follower = self.slave1 if 'follower' in act1 else self.slave2 rs2 = self.put(self.leader, tid, 1, 'testkey0', self.now(), 'testvalue0') self.assertIn('Put ok', rs2) rs3 = self.put(self.slave1, tid, 1, 'testkey0', self.now(), 'testvalue0') self.assertIn('Put failed', rs3) rs4 = self.put(leader_new, tid, 0, 'testkey0', self.now(), 'testvalue0') self.assertIn('Put ok', rs4) time.sleep(1) self.assertIn('testvalue0', self.scan(follower, tid, 0, 'testkey0', self.now(), 1)) self.ns_drop(self.ns_leader, name)
def test_delreplica_scenario(self): """ addreplica之前和delreplica之后,put到主节点的数据无法同步给副本 addreplica和再次addreplica之后,put到主节点的数据可以同步给副本 :return: """ rs1 = self.showopstatus(self.ns_leader) old_last_op_id = max(rs1.keys()) if rs1 != {} else 1 metadata_path = '{}/metadata.txt'.format(self.testpath) name = 'tname{}'.format(time.time()) if conf.multidimension is False: m = utils.gen_table_metadata( '"{}"'.format(name), None, 144000, 2, ('table_partition', '"{}"'.format(self.leader), '"0-2"', 'true')) else: m = utils.gen_table_metadata( '"{}"'.format(name), None, 144000, 2, ('table_partition', '"{}"'.format(self.leader), '"0-2"', 'true'), ('column_desc', '"merchant"', '"string"', 'true'), ('column_desc', '"amt"', '"double"', 'false'), ('column_desc', '"card"', '"string"', 'true'), ) utils.gen_table_metadata_file(m, metadata_path) rs2 = self.ns_create(self.ns_leader, metadata_path) self.assertIn('Create table ok', rs2) rs3 = self.showtable(self.ns_leader, name) tid = rs3.keys()[0][1] pid = 0 # put and addreplica self.multidimension_scan_vk = {'card': 'testkey0'} self.multidimension_vk = {'card': ('string:index', 'testkey0'), 'merchant': ('string:index', 'testvalue0'), 'amt': ('double', 1.1)} rs4 = self.put(self.leader, tid, pid, 'testkey0', self.now() + 10000, 'testvalue0') if rs4 == 'Put failed': infoLogger.error(' ') rs = self.showtable(self.ns_leader) tablestatus = self.parse_tb(rs, ' ', [0, 1, 2, 3], [4, 5, 6, 7, 8]) for status in tablestatus: infoLogger.info('{} = {}'.format(status, tablestatus[status])) infoLogger.error(' ') rs4 = self.put(self.leader, tid, pid, 'testkey0', self.now() + 10000, 'testvalue0') self.assertIn('Put ok', rs4) self.assertIn('Put ok', rs4) # makesnapshot rs5 = self.makesnapshot(self.ns_leader, name, pid, 'ns_client') self.assertIn('MakeSnapshot ok', rs5) time.sleep(2) self.showtablet(self.ns_leader) # addreplica by ns_client and put rs6 = self.addreplica(self.ns_leader, name, pid, 'ns_client', self.slave1) self.assertIn('AddReplica ok', rs6) last_op_id = max(self.showopstatus(self.ns_leader).keys()) self.assertTrue(old_last_op_id != last_op_id) last_opstatus = self.showopstatus(self.ns_leader)[last_op_id] self.assertIn('kAddReplicaOP', last_opstatus) self.multidimension_vk = {'card': ('string:index', 'testkey0'), 'merchant': ('string:index', 'testvalue1'), 'amt': ('double', 1.1)} rs7 = self.put(self.leader, tid, pid, 'testkey0', self.now() + 90000, 'testvalue1') self.assertIn('Put ok', rs7) time.sleep(5) # delreplica by ns_client and put rs8 = self.delreplica(self.ns_leader, name, pid, 'ns_client', self.slave1) self.assertIn('DelReplica ok', rs8) edps = '' for repeat in range(10): time.sleep(2) rs13 = self.showtable(self.ns_leader, name) flag = False for x in rs13: if x[3] == self.slave1: flag = True break if flag == False: edps = [x[3] for x in rs13] break self.assertFalse(self.slave1 in edps) self.multidimension_vk = {'card': ('string:index', 'testkey0'), 'merchant': ('string:index', 'testvalue2'), 'amt': ('double', 1.1)} rs9 = self.put(self.leader, tid, pid, 'testkey0', self.now() + 90000, 'testvalue2') self.assertIn('Put ok', rs9) # put after re-addreplica by client rs10 = self.addreplica(self.ns_leader, name, pid, 'ns_client', self.slave1) self.assertIn('AddReplica ok', rs10) self.multidimension_vk = {'card': ('string:index', 'testkey0'), 'merchant': ('string:index', 'testvalue3'), 'amt': ('double', 1.1)} rs11 = self.put(self.leader, tid, pid, 'testkey0', self.now() + 90000, 'testvalue3') self.assertIn('Put ok', rs11) time.sleep(5) rs12 = self.scan(self.slave1, tid, pid, 'testkey0', self.now() + 90000, 1) self.assertIn('testvalue0', rs12) self.assertIn('testvalue1', rs12) self.assertIn('testvalue2', rs12) self.assertIn('testvalue3', rs12) self.ns_drop(self.ns_leader, name)