def test_loadtable_by_binlog_success(self): """ 仅通过binlog loadtable :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) self.put(self.leader, self.tid, self.pid, 'testkey', self.now() - 10, 'testvalue') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs2) # 将table目录拷贝到新节点,删掉snapshot目录,保留binlog目录 self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) utils.exe_shell('rm -rf {}/db/{}_{}/snapshot/*'.format( self.slave1path, self.tid, self.pid)) rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid) self.assertIn('LoadTable ok', rs3) self.assertIn( 'testvalue', self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now(), 1)) rs4 = self.makesnapshot(self.slave1, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs4) mf = self.get_manifest(self.slave1path, self.tid, self.pid) self.assertEqual(mf['offset'], '1') self.assertTrue(mf['name']) self.assertEqual(mf['count'], '1')
def start_client(self, endpoint, role='tablet'): client_path = self.node_path_dict[endpoint] if role == 'tablet' or role == 'nameserver': conf = role else: pass cmd = '{}/fedb --flagfile={}/conf/{}.flags'.format( self.testpath, client_path, conf) infoLogger.info(cmd) args = shlex.split(cmd) need_start = False for _ in range(20): rs = utils.exe_shell('lsof -i:{}|grep -v "PID"'.format( endpoint.split(':')[1])) if 'fedb' not in rs: need_start = True time.sleep(1) subprocess.Popen( args, stdout=open('{}/info.log'.format(client_path), 'a'), stderr=open('{}/warning.log'.format(client_path), 'a')) else: time.sleep(1) rs = utils.exe_shell('lsof -i:{}|grep -v "PID"'.format( endpoint.split(':')[1])) if 'fedb' in rs: return True, need_start return False, need_start
def setUp(self): infoLogger.info('\nTEST CASE NAME: {} {} {}'.format( self, self._testMethodDoc, '\n' + '|' * 50 + ' SETUP STARTED ' + '|' * 50 + '\n')) try: self.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format( self.testpath)) nss = copy.deepcopy(conf.ns_endpoints) nss.remove(self.ns_leader) self.ns_slaver = nss[0] self.ns_leader_path = utils.exe_shell( 'tail -n 1 {}/ns_leader'.format(self.testpath)) self.tid = random.randint(1, 1000) self.pid = random.randint(1, 1000) if conf.cluster_mode == "cluster": self.clear_ns_table(self.ns_leader) else: for edp in conf.tb_endpoints: self.clear_tb_table(edp) #remote env self.alias = 'rem' self.ns_leader_r = utils.exe_shell( 'head -n 1 {}/ns_leader_remote'.format(self.testpath)) nss_r = copy.deepcopy(conf.ns_endpoints_r) nss_r.remove(self.ns_leader_r) self.ns_slaver_r = nss_r[0] self.ns_leader_path_r = utils.exe_shell( 'tail -n 1 {}/ns_leader_remote'.format(self.testpath)) self.clear_ns_table(self.ns_leader_r) for edp in conf.tb_endpoints_r: self.clear_tb_table(edp) except Exception as e: traceback.print_exc(file=sys.stdout) infoLogger.info('\n\n' + '=' * 50 + ' SETUP FINISHED ' + '=' * 50 + '\n')
def test_loadtable_snapshot_deleted(self): """ snapshot文件被删除,loadtable时无法load数据,log中有warning :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) self.put(self.leader, self.tid, self.pid, 'testkey', self.now(), 'testvalue') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs2) # 将table目录拷贝到新节点,并删掉sdb和binlog self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) utils.exe_shell('rm -f {}/db/{}_{}/snapshot/*.sdb'.format( self.slave1path, self.tid, self.pid)) utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format( self.slave1path, self.tid, self.pid)) rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid) self.assertIn('LoadTable ok', rs3) mf = self.get_manifest(self.slave1path, self.tid, self.pid) self.assertEqual(mf['offset'], '1') self.assertTrue(mf['name']) self.assertEqual(mf['count'], '1')
def stop_client(self, endpoint): port = endpoint.split(':')[1] cmd = "lsof -i:{}".format( port) + "|grep '(LISTEN)'|awk '{print $2}'|xargs kill -9" utils.exe_shell(cmd) rs = utils.exe_shell('lsof -i:{}|grep -v "PID"'.format(port)) if 'CLOSE_WAIT' in rs: time.sleep(2)
def kill(self, *endpoints): infoLogger.info(endpoints) port = '' for ep in endpoints: infoLogger.info(ep) port += ep.split(':')[1] + ' ' infoLogger.info(port) cmd = "for i in {};".format(port) + " do lsof -i:${i}|grep -v 'PID'|awk '{print $2}'|xargs kill -9;done" exe_shell(cmd) time.sleep(1)
def get_ns_leader(self, is_remote = False): cmd = ''; if not is_remote: cmd = "{}/fedb --zk_cluster={} --zk_root_path={} --role={} --interactive=false --cmd={}".format(self.test_path, self.zk_endpoint, "/onebox", "ns_client", "'showns'") else: cmd = "{}/fedb --zk_cluster={} --zk_root_path={} --role={} --interactive=false --cmd={}".format(self.test_path, self.zk_endpoint, "/remote", "ns_client", "'showns'") for i in xrange(5): result = exe_shell(cmd) rs_tb = result.split('\n') for line in rs_tb: if '-----------------------' in line or 'ns leader' in line: continue if 'leader' in line: ns_leader = line.strip().split(" ")[0].strip() self.ns_leader = ns_leader if not is_remote: exe_shell('echo "{}" > {}/ns_leader'.format(ns_leader, self.test_path)) exe_shell('echo "{}" >> {}/ns_leader'.format(self.ns_edp_path[ns_leader], self.test_path)) else: exe_shell('echo "{}" > {}/ns_leader_remote'.format(ns_leader, self.test_path)) exe_shell('echo "{}" >> {}/ns_leader_remote'.format(self.ns_edp_path[ns_leader] + 'remote', self.test_path)) return ns_leader time.sleep(2)
def test_put_slave_killed_while_leader_putting(self): """ 写数据过程中从节点挂掉,不影响主节点 重新启动后可以loadtable成功,数据与主节点一致 :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid, 144000, 8, 'true') self.assertIn('Create table ok', rs1) rs2 = self.create(self.slave1, 't', self.tid, self.pid, 144000, 8, 'false') self.assertIn('Create table ok', rs2) rs = self.addreplica(self.leader, self.tid, self.pid, 'client', self.slave1) self.assertIn('AddReplica ok', rs) def put(count): for i in range(0, count): self.put(self.leader, self.tid, self.pid, 'testkey', self.now() + 10000 + i, 'testvalue{}'.format(i)) def stop_client(endpoint): self.stop_client(endpoint) threads = [ threading.Thread(target=put, args=(20, )), threading.Thread(target=stop_client, args=(self.slave1, )) ] # 写入数据1s后节点挂掉 for t in threads: t.start() time.sleep(2) for t in threads: t.join() time.sleep(10) self.start_client(self.slave1) utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format( self.slave1path, self.tid, self.pid)) self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) rs4 = self.loadtable(self.slave1, 't', self.tid, self.pid, 144000, 8, 'false', self.slave1) self.assertIn('LoadTable ok', rs4) time.sleep(1) self.assertIn( 'testvalue19', self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now(), 1)) self.put(self.leader, self.tid, self.pid, 'testkey', self.now() + 10000, 'testvalue20') time.sleep(1) rs5 = self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now() * 2, 1) self.assertIn('testvalue20', rs5)
def update_conf(nodepath, conf_item, conf_value, role='client'): conf_file = '' if role == 'client': conf_file = 'tablet.flags' elif role == 'ns_client': conf_file = 'nameserver.flags' utils.exe_shell("sed -i '/{}/d' {}/conf/{}".format( conf_item, nodepath, conf_file)) if conf_value is not None: utils.exe_shell("sed -i '1i--{}={}' {}/conf/{}".format( conf_item, conf_value, nodepath, conf_file))
def setUp(self): infoLogger.info('\n' * 5 + 'TEST CASE NAME: ' + self._testMethodName + self._testMethodDoc) self.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format( self.testpath)) self.ns_leader_path = utils.exe_shell('tail -n 1 {}/ns_leader'.format( self.testpath)) self.tid = random.randint(1, 1000) self.pid = random.randint(1, 1000) self.clear_ns_table(self.ns_leader) infoLogger.info('\n\n' + '|' * 50 + ' SETUP FINISHED ' + '|' * 50 + '\n')
def get_new_ns_leader(self): nsc = NsCluster(conf.zk_endpoint, *(i for i in conf.ns_endpoints)) nsc.get_ns_leader() infoLogger.info(conf.ns_endpoints) nss = copy.deepcopy(conf.ns_endpoints) self.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format( self.testpath)) self.ns_leader_path = utils.exe_shell('tail -n 1 {}/ns_leader'.format( self.testpath)) self.node_path_dict[self.ns_leader] = utils.exe_shell( 'tail -n 1 {}/ns_leader'.format(self.testpath)) nss.remove(self.ns_leader) self.ns_slaver = nss[0] infoLogger.info("*" * 88) infoLogger.info("ns_leader: " + self.ns_leader) infoLogger.info("ns_slaver: " + self.ns_slaver) infoLogger.info("*" * 88)
def tearDownClass(cls): for edp in conf.tb_endpoints: utils.exe_shell('rm -rf {}/recycle/*'.format( cls.node_path_dict[edp])) utils.exe_shell('rm -rf {}/db/*'.format(cls.node_path_dict[edp])) for edp in conf.tb_endpoints_r: utils.exe_shell('rm -rf {}/recycle/*'.format( cls.node_path_dict_r[edp])) utils.exe_shell('rm -rf {}/db/*'.format(cls.node_path_dict_r[edp])) infoLogger.info('\n' + '=' * 50 + ' TEST {} FINISHED '.format(cls) + '=' * 50 + '\n' * 5)
def test_loadtable_by_binlog_ttl(self): """ binlog中包含过期数据 通过binlog loadtable,剔除过期数据 再makesnapshot可以成功 :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) for i in range(0, 4): # for multidimension test self.multidimension_vk = { 'card': ('string:index', 'testkey'), 'merchant': ('string:index', 'testvalue{}'.format(i)), 'amt': ('double', 1.1) } self.put(self.leader, self.tid, self.pid, 'testkey', self.now() - (100000000000 * (i % 2) + 1), 'testvalue{}'.format(i)) time.sleep(1) # 将table目录拷贝到新节点,删掉snapshot目录,保留binlog目录 self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) utils.exe_shell('rm -rf {}/db/{}_{}/snapshot/*'.format( self.slave1path, self.tid, self.pid)) rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid) self.assertIn('LoadTable ok', rs3) self.multidimension_scan_vk = { 'card': 'testkey' } # for multidimension test self.assertIn( 'testvalue0', self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now(), 1)) self.assertFalse('testvalue1' in self.scan( self.slave1, self.tid, self.pid, 'testkey', self.now(), 1)) rs4 = self.makesnapshot(self.slave1, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs4) mf = self.get_manifest(self.slave1path, self.tid, self.pid) self.assertEqual(mf['offset'], '4') self.assertTrue(mf['name']) self.assertEqual(mf['count'], '2')
def test_makesnapshot_manifest_deleted(self): """ MANIFEST文件被删除,makesnapshot无法做出新的snapshot文件,日志中有warning :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) for i in range(0, 6): self.put(self.leader, self.tid, self.pid, 'testkey', self.now() - (100000000000 * (i % 2) + 1), 'testvalue'*100) rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok' ,rs2) utils.exe_shell('rm -f {}/db/{}_{}/snapshot/MANIFEST'.format(self.leaderpath, self.tid, self.pid)) rs3 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok' ,rs3) time.sleep(1) # 新manifest的count=0,因为相当于没有新数据写入 # mf = self.get_manifest(self.leaderpath, self.tid, self.pid) # self.assertEqual(mf['offset'], '6') # self.assertTrue(mf['name']) # self.assertEqual(mf['count'], '0') # 再次写入数据后makesnapshot for i in range(0, 6): self.put(self.leader, self.tid, self.pid, 'testkey', self.now() - (100000000000 * (i % 2) + 1), 'testvalue') rs3 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok' ,rs3) mf = self.get_manifest(self.leaderpath, self.tid, self.pid) self.assertEqual(mf['offset'], '12') self.assertTrue(mf['name']) self.assertEqual(mf['count'], '3')
def test_makesnapshot_when_loading_table(self): """ loadtable过程中,无法makesnapshot :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) # for multidimension test self.multidimension_vk = {'card': ('string:index', 'v' * 120), 'merchant': ('string:index', 'v' * 120), 'amt': ('double', 1.1)} self.put_large_datas(3000, 8) # 将table目录拷贝到新节点 utils.exe_shell('cp -r {leaderpath}/db/{tid}_{pid} {slave1path}/db/'.format( leaderpath=self.leaderpath, tid=self.tid, pid=self.pid, slave1path=self.slave1path)) rs_list = [] def loadtable(endpoint): rs = self.loadtable(endpoint, 't', self.tid, self.pid) rs_list.append(rs) def makesnapshot(endpoint): rs = self.run_client(endpoint, 'makesnapshot {} {}'.format(self.tid, self.pid)) rs_list.append(rs) # 5个线程并发loadtable,最后只有1个线程是load ok的 threads = [threading.Thread( target=loadtable, args=(self.slave1,)), threading.Thread( target=makesnapshot, args=(self.slave1,))] for t in threads: time.sleep(0.0001) t.start() for t in threads: t.join() self.assertIn('LoadTable ok', rs_list) self.assertIn('Fail to MakeSnapshot', rs_list)
def test_makesnapshot_snapshot_name_mismatch(self): """ snapshot和MANIFEST名字不匹配,makesnapshot时日志中有warning,且无法生成新的snapshot :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) for i in range(0, 6): self.put(self.leader, self.tid, self.pid, 'testkey', self.now(), 'testvalue') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok' ,rs2) utils.exe_shell('mv {nodepath}/db/{tid}_{pid}/snapshot/*.sdb \ {nodepath}/db/{tid}_{pid}/snapshot/11111.sdb'.format( nodepath=self.leaderpath, tid=self.tid, pid=self.pid)) rs3 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok' ,rs3)
def test_loadtable_snapshot_name_mismatch(self): """ MANIFEST和snapshot文件名不匹配,loadtable时无法load数据,log中有warning :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) self.put(self.leader, self.tid, self.pid, 'testkey', self.now(), 'testvalue') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs2) # 将table目录拷贝到新节点,删除binlog,重命名sdb self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format( self.slave1path, self.tid, self.pid)) utils.exe_shell('mv {nodepath}/db/{tid}_{pid}/snapshot/*.sdb \ {nodepath}/db/{tid}_{pid}/snapshot/11111.sdb'.format( nodepath=self.slave1path, tid=self.tid, pid=self.pid)) rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid) self.assertIn('LoadTable ok', rs3)
def test_makesnapshot_snapshot_deleted(self): """ 删除snapshot后,makesnapshot无法做出新的snapshot文件,日志中有warning :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) for i in range(0, 6): self.put(self.leader, self.tid, self.pid, 'testkey', self.now() - (100000000000 * (i % 2) + 1), 'testvalue') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok' ,rs2) utils.exe_shell('rm -f {}/db/{}_{}/snapshot/*.sdb'.format(self.leaderpath, self.tid, self.pid)) rs3 = self.makesnapshot(self.leader, self.tid, self.pid) # should be failed? self.assertIn('MakeSnapshot ok' ,rs3)
def get_task_dict_by_opid(self, tname, opid): time.sleep(1) task_dict = collections.OrderedDict() cmd = "cat {}/warning.log |grep -a -A 10000 '{}'|grep -a \"op_id\[{}\]\"|grep task_type".format( self.ns_leader_path, tname, opid) \ + "|awk -F '\\\\[' '{print $3\"]\"$4\"]\"$5}'" \ "|awk -F '\\\\]' '{print $1\",\"$3\",\"$5}'" infoLogger.info(cmd) rs = utils.exe_shell(cmd).split('\n') infoLogger.info(rs) for x in rs: x = x.split(',') task_dict[(int(x[1]), x[2])] = x[0] self.task_dict = task_dict
def test_loadtable_manitest_deleted(self): """ MANIFEST文件被删除,loadtable时无法load数据,log中有warning :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) self.put(self.leader, self.tid, self.pid, 'testkey', self.now() - 10, 'testvalue') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs2) # 将table目录拷贝到新节点,并删掉manifest和binlog self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) utils.exe_shell('rm -f {}/db/{}_{}/snapshot/MANIFEST'.format( self.slave1path, self.tid, self.pid)) utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format( self.slave1path, self.tid, self.pid)) rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid) self.assertIn('LoadTable ok', rs3) rs4 = self.makesnapshot(self.slave1, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs4)
def clear_db(self): exe_shell('rm -rf {}/tablet*'.format(os.getenv('testpath')))
def get_ns_leader(self): return utils.exe_shell('cat {}/ns_leader'.format(self.testpath))
def clear_zk(self): exe_shell('rm -rf {}/data'.format(os.getenv('zkpath')))
def setUpClass(cls): infoLogger.info('\n' + '|' * 50 + ' TEST {} STARTED '.format(cls) + '|' * 50 + '\n') cls.welcome = 'Welcome to fedb with version {}\n'.format( os.getenv('rtidbver')) cls.testpath = os.getenv('testpath') cls.rtidb_path = os.getenv('rtidbpath') cls.conf_path = os.getenv('confpath') cls.data_path = os.getenv('datapath') cls.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format( cls.testpath)) cls.ns_leader_path = utils.exe_shell('tail -n 1 {}/ns_leader'.format( cls.testpath)) cls.ns_slaver = [i for i in conf.ns_endpoints if i != cls.ns_leader][0] cls.leader, cls.slave1, cls.slave2 = (i for i in conf.tb_endpoints) cls.multidimension = conf.multidimension cls.multidimension_vk = conf.multidimension_vk cls.multidimension_scan_vk = conf.multidimension_scan_vk cls.failfast = conf.failfast cls.ns_path_dict = { conf.ns_endpoints[0]: cls.testpath + '/ns1', conf.ns_endpoints[1]: cls.testpath + '/ns2' } cls.node_path_dict = { cls.leader: cls.testpath + '/tablet1', cls.slave1: cls.testpath + '/tablet2', cls.slave2: cls.testpath + '/tablet3', cls.ns_leader: cls.ns_path_dict[cls.ns_leader], cls.ns_slaver: cls.ns_path_dict[cls.ns_slaver] } cls.leaderpath = cls.node_path_dict[cls.leader] cls.slave1path = cls.node_path_dict[cls.slave1] cls.slave2path = cls.node_path_dict[cls.slave2] infoLogger.info('*' * 88) infoLogger.info([i for i in conf.ns_endpoints]) infoLogger.info(cls.ns_slaver) infoLogger.info(conf.cluster_mode) # remote env cls.ns_leader_r = utils.exe_shell( 'head -n 1 {}/ns_leader_remote'.format(cls.testpath)) cls.ns_leader_path_r = utils.exe_shell( 'tail -n 1 {}/ns_leader_remote'.format(cls.testpath)) cls.ns_slaver_r = [ i for i in conf.ns_endpoints_r if i != cls.ns_leader_r ][0] cls.leader_r, cls.slave1_r, cls.slave2_r = ( i for i in conf.tb_endpoints_r) cls.ns_path_dict_r = { conf.ns_endpoints_r[0]: cls.testpath + '/ns1remote', conf.ns_endpoints_r[1]: cls.testpath + '/ns2remote' } cls.node_path_dict_r = { cls.leader_r: cls.testpath + '/tablet1remote', cls.slave1_r: cls.testpath + '/tablet2remote', cls.slave2_r: cls.testpath + '/tablet3remote', cls.ns_leader_r: cls.ns_path_dict_r[cls.ns_leader_r], cls.ns_slaver_r: cls.ns_path_dict_r[cls.ns_slaver_r] } infoLogger.info('*' * 88) infoLogger.info([i for i in conf.ns_endpoints_r]) infoLogger.info(cls.ns_slaver_r)
def cp_db(self, from_node, to_node, tid, pid): utils.exe_shell( 'cp -r {from_node}/db/{tid}_{pid} {to_node}/db/'.format( from_node=from_node, tid=tid, pid=pid, to_node=to_node))
def execute_gc(self, endpoint, tid, pid): cmd = "curl -d \'{\"tid\":%s, \"pid\":%s}\' http://%s/TabletServer/ExecuteGc" % ( tid, pid, endpoint) utils.exe_shell(cmd) time.sleep(2)
def run_client(self, endpoint, cmd, role='client'): cmd = cmd.strip() rs = utils.exe_shell( '{} --endpoint={} --role={} --interactive=false --request_timeout_ms=200000 --cmd="{}"' .format(self.rtidb_path, endpoint, role, cmd)) return rs.replace(self.welcome, '').replace('>', '')
def start(self, endpoints, is_remote=False): tbconfpath = os.getenv('tbconfpath') i = 0 test_path = os.getenv('testpath') for ep in endpoints: i += 1 if not is_remote: tb_path = test_path + '/tablet{}'.format(i) else: tb_path = test_path + '/tablet{}'.format(i) + 'remote' esc_tb_path = tb_path.replace("/", "\/") exe_shell('export ') exe_shell('rm -rf {}/*'.format(tb_path)) rtidb_flags = '{}/conf/tablet.flags'.format(tb_path) exe_shell('mkdir -p {}/conf'.format(tb_path)) exe_shell( 'cat {} | egrep -v "endpoint|log_level|gc_interval|log_dir" > ' '{}'.format(tbconfpath, rtidb_flags)) exe_shell("sed -i '1a --endpoint='{} {}".format(ep, rtidb_flags)) exe_shell("sed -i '1a --gc_interval=1' {}".format(rtidb_flags)) exe_shell("sed -i 's/--db_root_path=.*/--db_root_path={}\/db/' {}". format(esc_tb_path, rtidb_flags)) exe_shell("sed -i '1a --zk_cluster='{} {}".format( self.zk_endpoint, rtidb_flags)) exe_shell( "sed -i 's/--recycle_bin_root_path=.*/--recycle_bin_root_path={}\/recycle/' {}" .format(esc_tb_path, rtidb_flags)) exe_shell("echo '--log_level={}' >> {}".format( conf.rtidb_log_info, rtidb_flags)) exe_shell("echo '--stream_close_wait_time_ms=10' >> {}".format( rtidb_flags)) exe_shell( "echo '--stream_bandwidth_limit=0' >> {}".format(rtidb_flags)) if not is_remote: exe_shell( "echo '--zk_root_path=/onebox' >> {}".format(rtidb_flags)) else: exe_shell( "echo '--zk_root_path=/remote' >> {}".format(rtidb_flags)) exe_shell( "echo '--zk_keep_alive_check_interval=500000' >> {}".format( rtidb_flags)) exe_shell("echo '--gc_safe_offset=0' >> {}".format(rtidb_flags)) exe_shell("echo '--binlog_sync_to_disk_interval=10' >> {}".format( rtidb_flags)) exe_shell( "echo '--binlog_sync_wait_time=10' >> {}".format(rtidb_flags)) exe_shell( "echo '--zk_session_timeout=2000' >> {}".format(rtidb_flags)) exe_shell("echo '--make_snapshot_threshold_offset=0' >> {}".format( rtidb_flags)) exe_shell("echo '--ssd_root_path={}/ssd_db/' >> {}".format( tb_path, rtidb_flags)) exe_shell("echo '--hdd_root_path={}/hdd_db/' >> {}".format( tb_path, rtidb_flags)) exe_shell( "echo '--recycle_ssd_bin_root_path={}/ssd_recycle/' >> {}". format(tb_path, rtidb_flags)) exe_shell( "echo '--recycle_hdd_bin_root_path={}/hdd_recycle/' >> {}". format(tb_path, rtidb_flags)) exe_shell("ulimit -c unlimited") cmd = '{}/fedb --flagfile={}/conf/tablet.flags'.format( test_path, tb_path) infoLogger.info('start fedb: {}'.format(cmd)) args = shlex.split(cmd) started = [] for _ in range(5): rs = exe_shell('lsof -i:{}|grep -v "PID"'.format( ep.split(':')[1])) if 'fedb' not in rs: time.sleep(2) subprocess.Popen( args, stdout=open('{}/info.log'.format(tb_path), 'a'), stderr=open('{}/warning.log'.format(tb_path), 'a')) else: started.append(True) break return started
def start(self, is_remote, *endpoints): nsconfpath = os.getenv('nsconfpath') i = 0 for ep in endpoints: i += 1 if not is_remote: ns_path = self.ns_edp_path[ep] else: ns_path = self.ns_edp_path[ep] + 'remote' nameserver_flags = '{}/conf/nameserver.flags'.format(ns_path) exe_shell('mkdir -p {}/conf'.format(ns_path)) exe_shell('touch {}'.format(nameserver_flags)) exe_shell("echo '--log_level={}' >> {}".format(conf.rtidb_log_info, nameserver_flags)) exe_shell("echo '--endpoint='{} >> {}".format(ep, nameserver_flags)) exe_shell("echo '--role=nameserver' >> {}".format(nameserver_flags)) exe_shell("echo '--zk_cluster='{} >> {}".format(self.zk_endpoint, nameserver_flags)) if not is_remote: exe_shell("echo '--zk_root_path=/onebox' >> {}".format(nameserver_flags)) else: exe_shell("echo '--zk_root_path=/remote' >> {}".format(nameserver_flags)) exe_shell("echo '--auto_failover=false' >> {}".format(nameserver_flags)) exe_shell("echo '--get_task_status_interval=100' >> {}".format(nameserver_flags)) exe_shell("echo '--name_server_task_pool_size=10' >> {}".format(nameserver_flags)) exe_shell("echo '--zk_keep_alive_check_interval=500000' >> {}".format(nameserver_flags)) exe_shell("echo '--tablet_offline_check_interval=10' >> {}".format(nameserver_flags)) exe_shell("echo '--tablet_heartbeat_timeout=0' >> {}".format(nameserver_flags)) exe_shell("echo '--request_timeout_ms=100000' >> {}".format(nameserver_flags)) exe_shell("echo '--name_server_task_concurrency=8' >> {}".format(nameserver_flags)) if is_remote: exe_shell("echo '--get_replica_status_interval=600000' >> {}".format(nameserver_flags)) exe_shell("echo '--zk_session_timeout=2000' >> {}".format(nameserver_flags)) exe_shell("ulimit -c unlimited") cmd = '{}/fedb --flagfile={}'.format(self.test_path, nameserver_flags) infoLogger.info('start fedb: {}'.format(cmd)) args = shlex.split(cmd) started = [] for _ in range(5): rs = exe_shell('lsof -i:{}|grep -v "PID"'.format(ep.split(':')[1])) if 'fedb' not in rs: time.sleep(2) subprocess.Popen(args,stdout=open('{}/info.log'.format(ns_path), 'a'), stderr=open('{}/warning.log'.format(ns_path), 'a')) else: started.append(True) break return started
def test_loadtable_by_binlog_after_makesnapshots(self): """ 多次makesnapshot后,可以通过binlog loadtable,且数据完整 :return: """ rs1 = self.create(self.leader, 't', self.tid, self.pid) self.assertIn('Create table ok', rs1) for i in range(0, 3): # for multidimension test self.multidimension_vk = { 'card': ('string:index', 'testkey1'), 'merchant': ('string:index', 'testvalue1'), 'amt': ('double', 1.1) } self.put(self.leader, self.tid, self.pid, 'testkey1', self.now() - i, 'testvalue1') rs2 = self.makesnapshot(self.leader, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs2) for i in range(0, 3): # for multidimension test self.multidimension_vk = { 'card': ('string:index', 'testkey2'), 'merchant': ('string:index', 'testvalue2'), 'amt': ('double', 1.1) } self.put(self.leader, self.tid, self.pid, 'testkey2', self.now() - i, 'testvalue2') # 将table目录拷贝到新节点,删掉snapshot目录,保留binlog目录 self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid) utils.exe_shell('rm -rf {}/db/{}_{}/snapshot/*'.format( self.slave1path, self.tid, self.pid)) rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid) self.assertIn('LoadTable ok', rs3) self.multidimension_scan_vk = { 'card': 'testkey1' } # for multidimension test self.assertIn( 'testvalue1', self.scan(self.slave1, self.tid, self.pid, 'testkey1', self.now(), 1)) self.multidimension_scan_vk = { 'card': 'testkey2' } # for multidimension test self.assertIn( 'testvalue2', self.scan(self.slave1, self.tid, self.pid, 'testkey2', self.now(), 1)) rs4 = self.makesnapshot(self.slave1, self.tid, self.pid) self.assertIn('MakeSnapshot ok', rs4) mf = self.get_manifest(self.slave1path, self.tid, self.pid) self.assertEqual(mf['offset'], '6') self.assertTrue(mf['name']) self.assertEqual(mf['count'], '6') table_status = self.get_table_status(self.leader, self.tid, self.pid) self.assertEqual( table_status[:6], ['6', 'kTableLeader', 'kTableNormal', 'true', '144000min', '0s'])