def test_loadtable_by_binlog_success(self):
"""
仅通过binlog loadtable
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
self.put(self.leader, self.tid, self.pid, 'testkey',
self.now() - 10, 'testvalue')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs2)
# 将table目录拷贝到新节点，删掉snapshot目录，保留binlog目录
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
utils.exe_shell('rm -rf {}/db/{}_{}/snapshot/*'.format(
self.slave1path, self.tid, self.pid))
rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs3)
self.assertIn(
'testvalue',
self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now(),
1))
rs4 = self.makesnapshot(self.slave1, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs4)
mf = self.get_manifest(self.slave1path, self.tid, self.pid)
self.assertEqual(mf['offset'], '1')
self.assertTrue(mf['name'])
self.assertEqual(mf['count'], '1')
def start_client(self, endpoint, role='tablet'):
client_path = self.node_path_dict[endpoint]
if role == 'tablet' or role == 'nameserver':
conf = role
else:
pass
cmd = '{}/fedb --flagfile={}/conf/{}.flags'.format(
self.testpath, client_path, conf)
infoLogger.info(cmd)
args = shlex.split(cmd)
need_start = False
for _ in range(20):
rs = utils.exe_shell('lsof -i:{}|grep -v "PID"'.format(
endpoint.split(':')[1]))
if 'fedb' not in rs:
need_start = True
time.sleep(1)
subprocess.Popen(
args,
stdout=open('{}/info.log'.format(client_path), 'a'),
stderr=open('{}/warning.log'.format(client_path), 'a'))
else:
time.sleep(1)
rs = utils.exe_shell('lsof -i:{}|grep -v "PID"'.format(
endpoint.split(':')[1]))
if 'fedb' in rs:
return True, need_start
return False, need_start
def setUp(self):
infoLogger.info('\nTEST CASE NAME: {} {} {}'.format(
self, self._testMethodDoc,
'\n' + '|' * 50 + ' SETUP STARTED ' + '|' * 50 + '\n'))
try:
self.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format(
self.testpath))
nss = copy.deepcopy(conf.ns_endpoints)
nss.remove(self.ns_leader)
self.ns_slaver = nss[0]
self.ns_leader_path = utils.exe_shell(
'tail -n 1 {}/ns_leader'.format(self.testpath))
self.tid = random.randint(1, 1000)
self.pid = random.randint(1, 1000)
if conf.cluster_mode == "cluster":
self.clear_ns_table(self.ns_leader)
else:
for edp in conf.tb_endpoints:
self.clear_tb_table(edp)
#remote env
self.alias = 'rem'
self.ns_leader_r = utils.exe_shell(
'head -n 1 {}/ns_leader_remote'.format(self.testpath))
nss_r = copy.deepcopy(conf.ns_endpoints_r)
nss_r.remove(self.ns_leader_r)
self.ns_slaver_r = nss_r[0]
self.ns_leader_path_r = utils.exe_shell(
'tail -n 1 {}/ns_leader_remote'.format(self.testpath))
self.clear_ns_table(self.ns_leader_r)
for edp in conf.tb_endpoints_r:
self.clear_tb_table(edp)
except Exception as e:
traceback.print_exc(file=sys.stdout)
infoLogger.info('\n\n' + '=' * 50 + ' SETUP FINISHED ' + '=' * 50 +
'\n')
def test_loadtable_snapshot_deleted(self):
"""
snapshot文件被删除，loadtable时无法load数据，log中有warning
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
self.put(self.leader, self.tid, self.pid, 'testkey', self.now(),
'testvalue')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs2)
# 将table目录拷贝到新节点，并删掉sdb和binlog
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
utils.exe_shell('rm -f {}/db/{}_{}/snapshot/*.sdb'.format(
self.slave1path, self.tid, self.pid))
utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format(
self.slave1path, self.tid, self.pid))
rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs3)
mf = self.get_manifest(self.slave1path, self.tid, self.pid)
self.assertEqual(mf['offset'], '1')
self.assertTrue(mf['name'])
self.assertEqual(mf['count'], '1')
def stop_client(self, endpoint):
port = endpoint.split(':')[1]
cmd = "lsof -i:{}".format(
port) + "|grep '(LISTEN)'|awk '{print $2}'|xargs kill -9"
utils.exe_shell(cmd)
rs = utils.exe_shell('lsof -i:{}|grep -v "PID"'.format(port))
if 'CLOSE_WAIT' in rs:
time.sleep(2)
def kill(self, *endpoints):
infoLogger.info(endpoints)
port = ''
for ep in endpoints:
infoLogger.info(ep)
port += ep.split(':')[1] + ' '
infoLogger.info(port)
cmd = "for i in {};".format(port) + " do lsof -i:${i}|grep -v 'PID'|awk '{print $2}'|xargs kill -9;done"
exe_shell(cmd)
time.sleep(1)
def get_ns_leader(self, is_remote = False):
cmd = '';
if not is_remote:
cmd = "{}/fedb --zk_cluster={} --zk_root_path={} --role={} --interactive=false --cmd={}".format(self.test_path,
self.zk_endpoint, "/onebox", "ns_client", "'showns'")
else:
cmd = "{}/fedb --zk_cluster={} --zk_root_path={} --role={} --interactive=false --cmd={}".format(self.test_path,
self.zk_endpoint, "/remote", "ns_client", "'showns'")
for i in xrange(5):
result = exe_shell(cmd)
rs_tb = result.split('\n')
for line in rs_tb:
if '-----------------------' in line or 'ns leader' in line:
continue
if 'leader' in line:
ns_leader = line.strip().split(" ")[0].strip()
self.ns_leader = ns_leader
if not is_remote:
exe_shell('echo "{}" > {}/ns_leader'.format(ns_leader, self.test_path))
exe_shell('echo "{}" >> {}/ns_leader'.format(self.ns_edp_path[ns_leader], self.test_path))
else:
exe_shell('echo "{}" > {}/ns_leader_remote'.format(ns_leader, self.test_path))
exe_shell('echo "{}" >> {}/ns_leader_remote'.format(self.ns_edp_path[ns_leader] + 'remote', self.test_path))
return ns_leader
time.sleep(2)
def test_put_slave_killed_while_leader_putting(self):
"""
写数据过程中从节点挂掉，不影响主节点
重新启动后可以loadtable成功，数据与主节点一致
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid, 144000, 8,
'true')
self.assertIn('Create table ok', rs1)
rs2 = self.create(self.slave1, 't', self.tid, self.pid, 144000, 8,
'false')
self.assertIn('Create table ok', rs2)
rs = self.addreplica(self.leader, self.tid, self.pid, 'client',
self.slave1)
self.assertIn('AddReplica ok', rs)
def put(count):
for i in range(0, count):
self.put(self.leader, self.tid, self.pid, 'testkey',
self.now() + 10000 + i, 'testvalue{}'.format(i))
def stop_client(endpoint):
self.stop_client(endpoint)
threads = [
threading.Thread(target=put, args=(20, )),
threading.Thread(target=stop_client, args=(self.slave1, ))
]
# 写入数据1s后节点挂掉
for t in threads:
t.start()
time.sleep(2)
for t in threads:
t.join()
time.sleep(10)
self.start_client(self.slave1)
utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format(
self.slave1path, self.tid, self.pid))
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
rs4 = self.loadtable(self.slave1, 't', self.tid, self.pid, 144000, 8,
'false', self.slave1)
self.assertIn('LoadTable ok', rs4)
time.sleep(1)
self.assertIn(
'testvalue19',
self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now(),
1))
self.put(self.leader, self.tid, self.pid, 'testkey',
self.now() + 10000, 'testvalue20')
time.sleep(1)
rs5 = self.scan(self.slave1, self.tid, self.pid, 'testkey',
self.now() * 2, 1)
self.assertIn('testvalue20', rs5)
def update_conf(nodepath, conf_item, conf_value, role='client'):
conf_file = ''
if role == 'client':
conf_file = 'tablet.flags'
elif role == 'ns_client':
conf_file = 'nameserver.flags'
utils.exe_shell("sed -i '/{}/d' {}/conf/{}".format(
conf_item, nodepath, conf_file))
if conf_value is not None:
utils.exe_shell("sed -i '1i--{}={}' {}/conf/{}".format(
conf_item, conf_value, nodepath, conf_file))
def setUp(self):
infoLogger.info('\n' * 5 + 'TEST CASE NAME: ' + self._testMethodName +
self._testMethodDoc)
self.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format(
self.testpath))
self.ns_leader_path = utils.exe_shell('tail -n 1 {}/ns_leader'.format(
self.testpath))
self.tid = random.randint(1, 1000)
self.pid = random.randint(1, 1000)
self.clear_ns_table(self.ns_leader)
infoLogger.info('\n\n' + '|' * 50 + ' SETUP FINISHED ' + '|' * 50 +
'\n')
def get_new_ns_leader(self):
nsc = NsCluster(conf.zk_endpoint, *(i for i in conf.ns_endpoints))
nsc.get_ns_leader()
infoLogger.info(conf.ns_endpoints)
nss = copy.deepcopy(conf.ns_endpoints)
self.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format(
self.testpath))
self.ns_leader_path = utils.exe_shell('tail -n 1 {}/ns_leader'.format(
self.testpath))
self.node_path_dict[self.ns_leader] = utils.exe_shell(
'tail -n 1 {}/ns_leader'.format(self.testpath))
nss.remove(self.ns_leader)
self.ns_slaver = nss[0]
infoLogger.info("*" * 88)
infoLogger.info("ns_leader: " + self.ns_leader)
infoLogger.info("ns_slaver: " + self.ns_slaver)
infoLogger.info("*" * 88)
def tearDownClass(cls):
for edp in conf.tb_endpoints:
utils.exe_shell('rm -rf {}/recycle/*'.format(
cls.node_path_dict[edp]))
utils.exe_shell('rm -rf {}/db/*'.format(cls.node_path_dict[edp]))
for edp in conf.tb_endpoints_r:
utils.exe_shell('rm -rf {}/recycle/*'.format(
cls.node_path_dict_r[edp]))
utils.exe_shell('rm -rf {}/db/*'.format(cls.node_path_dict_r[edp]))
infoLogger.info('\n' + '=' * 50 + ' TEST {} FINISHED '.format(cls) +
'=' * 50 + '\n' * 5)
def test_loadtable_by_binlog_ttl(self):
"""
binlog中包含过期数据
通过binlog loadtable，剔除过期数据
再makesnapshot可以成功
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
for i in range(0, 4):
# for multidimension test
self.multidimension_vk = {
'card': ('string:index', 'testkey'),
'merchant': ('string:index', 'testvalue{}'.format(i)),
'amt': ('double', 1.1)
}
self.put(self.leader, self.tid, self.pid, 'testkey',
self.now() - (100000000000 * (i % 2) + 1),
'testvalue{}'.format(i))
time.sleep(1)
# 将table目录拷贝到新节点，删掉snapshot目录，保留binlog目录
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
utils.exe_shell('rm -rf {}/db/{}_{}/snapshot/*'.format(
self.slave1path, self.tid, self.pid))
rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs3)
self.multidimension_scan_vk = {
'card': 'testkey'
} # for multidimension test
self.assertIn(
'testvalue0',
self.scan(self.slave1, self.tid, self.pid, 'testkey', self.now(),
1))
self.assertFalse('testvalue1' in self.scan(
self.slave1, self.tid, self.pid, 'testkey', self.now(), 1))
rs4 = self.makesnapshot(self.slave1, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs4)
mf = self.get_manifest(self.slave1path, self.tid, self.pid)
self.assertEqual(mf['offset'], '4')
self.assertTrue(mf['name'])
self.assertEqual(mf['count'], '2')
def test_makesnapshot_manifest_deleted(self):
"""
MANIFEST文件被删除，makesnapshot无法做出新的snapshot文件，日志中有warning
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
for i in range(0, 6):
self.put(self.leader,
self.tid,
self.pid,
'testkey',
self.now() - (100000000000 * (i % 2) + 1),
'testvalue'*100)
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs2)
utils.exe_shell('rm -f {}/db/{}_{}/snapshot/MANIFEST'.format(self.leaderpath, self.tid, self.pid))
rs3 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs3)
time.sleep(1)
# 新manifest的count=0，因为相当于没有新数据写入
# mf = self.get_manifest(self.leaderpath, self.tid, self.pid)
# self.assertEqual(mf['offset'], '6')
# self.assertTrue(mf['name'])
# self.assertEqual(mf['count'], '0')
# 再次写入数据后makesnapshot
for i in range(0, 6):
self.put(self.leader,
self.tid,
self.pid,
'testkey',
self.now() - (100000000000 * (i % 2) + 1),
'testvalue')
rs3 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs3)
mf = self.get_manifest(self.leaderpath, self.tid, self.pid)
self.assertEqual(mf['offset'], '12')
self.assertTrue(mf['name'])
self.assertEqual(mf['count'], '3')
def test_makesnapshot_when_loading_table(self):
"""
loadtable过程中，无法makesnapshot
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
# for multidimension test
self.multidimension_vk = {'card': ('string:index', 'v' * 120),
'merchant': ('string:index', 'v' * 120),
'amt': ('double', 1.1)}
self.put_large_datas(3000, 8)
# 将table目录拷贝到新节点
utils.exe_shell('cp -r {leaderpath}/db/{tid}_{pid} {slave1path}/db/'.format(
leaderpath=self.leaderpath, tid=self.tid, pid=self.pid, slave1path=self.slave1path))
rs_list = []
def loadtable(endpoint):
rs = self.loadtable(endpoint, 't', self.tid, self.pid)
rs_list.append(rs)
def makesnapshot(endpoint):
rs = self.run_client(endpoint, 'makesnapshot {} {}'.format(self.tid, self.pid))
rs_list.append(rs)
# 5个线程并发loadtable，最后只有1个线程是load ok的
threads = [threading.Thread(
target=loadtable, args=(self.slave1,)), threading.Thread(
target=makesnapshot, args=(self.slave1,))]
for t in threads:
time.sleep(0.0001)
t.start()
for t in threads:
t.join()
self.assertIn('LoadTable ok', rs_list)
self.assertIn('Fail to MakeSnapshot', rs_list)
def test_makesnapshot_snapshot_name_mismatch(self):
"""
snapshot和MANIFEST名字不匹配，makesnapshot时日志中有warning，且无法生成新的snapshot
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
for i in range(0, 6):
self.put(self.leader,
self.tid,
self.pid,
'testkey',
self.now(),
'testvalue')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs2)
utils.exe_shell('mv {nodepath}/db/{tid}_{pid}/snapshot/*.sdb \
{nodepath}/db/{tid}_{pid}/snapshot/11111.sdb'.format(
nodepath=self.leaderpath, tid=self.tid, pid=self.pid))
rs3 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs3)
def test_loadtable_snapshot_name_mismatch(self):
"""
MANIFEST和snapshot文件名不匹配，loadtable时无法load数据，log中有warning
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
self.put(self.leader, self.tid, self.pid, 'testkey', self.now(),
'testvalue')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs2)
# 将table目录拷贝到新节点，删除binlog，重命名sdb
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format(
self.slave1path, self.tid, self.pid))
utils.exe_shell('mv {nodepath}/db/{tid}_{pid}/snapshot/*.sdb \
{nodepath}/db/{tid}_{pid}/snapshot/11111.sdb'.format(
nodepath=self.slave1path, tid=self.tid, pid=self.pid))
rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs3)
def test_makesnapshot_snapshot_deleted(self):
"""
删除snapshot后，makesnapshot无法做出新的snapshot文件，日志中有warning
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
for i in range(0, 6):
self.put(self.leader,
self.tid,
self.pid,
'testkey',
self.now() - (100000000000 * (i % 2) + 1),
'testvalue')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok' ,rs2)
utils.exe_shell('rm -f {}/db/{}_{}/snapshot/*.sdb'.format(self.leaderpath, self.tid, self.pid))
rs3 = self.makesnapshot(self.leader, self.tid, self.pid)
# should be failed?
self.assertIn('MakeSnapshot ok' ,rs3)
def get_task_dict_by_opid(self, tname, opid):
time.sleep(1)
task_dict = collections.OrderedDict()
cmd = "cat {}/warning.log |grep -a -A 10000 '{}'|grep -a \"op_id\[{}\]\"|grep task_type".format(
self.ns_leader_path, tname, opid) \
+ "|awk -F '\\\\[' '{print $3\"]\"$4\"]\"$5}'" \
"|awk -F '\\\\]' '{print $1\",\"$3\",\"$5}'"
infoLogger.info(cmd)
rs = utils.exe_shell(cmd).split('\n')
infoLogger.info(rs)
for x in rs:
x = x.split(',')
task_dict[(int(x[1]), x[2])] = x[0]
self.task_dict = task_dict
def test_loadtable_manitest_deleted(self):
"""
MANIFEST文件被删除，loadtable时无法load数据，log中有warning
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
self.put(self.leader, self.tid, self.pid, 'testkey',
self.now() - 10, 'testvalue')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs2)
# 将table目录拷贝到新节点，并删掉manifest和binlog
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
utils.exe_shell('rm -f {}/db/{}_{}/snapshot/MANIFEST'.format(
self.slave1path, self.tid, self.pid))
utils.exe_shell('rm -rf {}/db/{}_{}/binlog'.format(
self.slave1path, self.tid, self.pid))
rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs3)
rs4 = self.makesnapshot(self.slave1, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs4)
def clear_db(self):
exe_shell('rm -rf {}/tablet*'.format(os.getenv('testpath')))
def get_ns_leader(self):
return utils.exe_shell('cat {}/ns_leader'.format(self.testpath))
def clear_zk(self):
exe_shell('rm -rf {}/data'.format(os.getenv('zkpath')))
def setUpClass(cls):
infoLogger.info('\n' + '|' * 50 + ' TEST {} STARTED '.format(cls) +
'|' * 50 + '\n')
cls.welcome = 'Welcome to fedb with version {}\n'.format(
os.getenv('rtidbver'))
cls.testpath = os.getenv('testpath')
cls.rtidb_path = os.getenv('rtidbpath')
cls.conf_path = os.getenv('confpath')
cls.data_path = os.getenv('datapath')
cls.ns_leader = utils.exe_shell('head -n 1 {}/ns_leader'.format(
cls.testpath))
cls.ns_leader_path = utils.exe_shell('tail -n 1 {}/ns_leader'.format(
cls.testpath))
cls.ns_slaver = [i for i in conf.ns_endpoints if i != cls.ns_leader][0]
cls.leader, cls.slave1, cls.slave2 = (i for i in conf.tb_endpoints)
cls.multidimension = conf.multidimension
cls.multidimension_vk = conf.multidimension_vk
cls.multidimension_scan_vk = conf.multidimension_scan_vk
cls.failfast = conf.failfast
cls.ns_path_dict = {
conf.ns_endpoints[0]: cls.testpath + '/ns1',
conf.ns_endpoints[1]: cls.testpath + '/ns2'
}
cls.node_path_dict = {
cls.leader: cls.testpath + '/tablet1',
cls.slave1: cls.testpath + '/tablet2',
cls.slave2: cls.testpath + '/tablet3',
cls.ns_leader: cls.ns_path_dict[cls.ns_leader],
cls.ns_slaver: cls.ns_path_dict[cls.ns_slaver]
}
cls.leaderpath = cls.node_path_dict[cls.leader]
cls.slave1path = cls.node_path_dict[cls.slave1]
cls.slave2path = cls.node_path_dict[cls.slave2]
infoLogger.info('*' * 88)
infoLogger.info([i for i in conf.ns_endpoints])
infoLogger.info(cls.ns_slaver)
infoLogger.info(conf.cluster_mode)
# remote env
cls.ns_leader_r = utils.exe_shell(
'head -n 1 {}/ns_leader_remote'.format(cls.testpath))
cls.ns_leader_path_r = utils.exe_shell(
'tail -n 1 {}/ns_leader_remote'.format(cls.testpath))
cls.ns_slaver_r = [
i for i in conf.ns_endpoints_r if i != cls.ns_leader_r
][0]
cls.leader_r, cls.slave1_r, cls.slave2_r = (
i for i in conf.tb_endpoints_r)
cls.ns_path_dict_r = {
conf.ns_endpoints_r[0]: cls.testpath + '/ns1remote',
conf.ns_endpoints_r[1]: cls.testpath + '/ns2remote'
}
cls.node_path_dict_r = {
cls.leader_r: cls.testpath + '/tablet1remote',
cls.slave1_r: cls.testpath + '/tablet2remote',
cls.slave2_r: cls.testpath + '/tablet3remote',
cls.ns_leader_r: cls.ns_path_dict_r[cls.ns_leader_r],
cls.ns_slaver_r: cls.ns_path_dict_r[cls.ns_slaver_r]
}
infoLogger.info('*' * 88)
infoLogger.info([i for i in conf.ns_endpoints_r])
infoLogger.info(cls.ns_slaver_r)
def cp_db(self, from_node, to_node, tid, pid):
utils.exe_shell(
'cp -r {from_node}/db/{tid}_{pid} {to_node}/db/'.format(
from_node=from_node, tid=tid, pid=pid, to_node=to_node))
def execute_gc(self, endpoint, tid, pid):
cmd = "curl -d \'{\"tid\":%s, \"pid\":%s}\' http://%s/TabletServer/ExecuteGc" % (
tid, pid, endpoint)
utils.exe_shell(cmd)
time.sleep(2)
def run_client(self, endpoint, cmd, role='client'):
cmd = cmd.strip()
rs = utils.exe_shell(
'{} --endpoint={} --role={} --interactive=false --request_timeout_ms=200000 --cmd="{}"'
.format(self.rtidb_path, endpoint, role, cmd))
return rs.replace(self.welcome, '').replace('>', '')
def start(self, endpoints, is_remote=False):
tbconfpath = os.getenv('tbconfpath')
i = 0
test_path = os.getenv('testpath')
for ep in endpoints:
i += 1
if not is_remote:
tb_path = test_path + '/tablet{}'.format(i)
else:
tb_path = test_path + '/tablet{}'.format(i) + 'remote'
esc_tb_path = tb_path.replace("/", "\/")
exe_shell('export ')
exe_shell('rm -rf {}/*'.format(tb_path))
rtidb_flags = '{}/conf/tablet.flags'.format(tb_path)
exe_shell('mkdir -p {}/conf'.format(tb_path))
exe_shell(
'cat {} | egrep -v "endpoint|log_level|gc_interval|log_dir" > '
'{}'.format(tbconfpath, rtidb_flags))
exe_shell("sed -i '1a --endpoint='{} {}".format(ep, rtidb_flags))
exe_shell("sed -i '1a --gc_interval=1' {}".format(rtidb_flags))
exe_shell("sed -i 's/--db_root_path=.*/--db_root_path={}\/db/' {}".
format(esc_tb_path, rtidb_flags))
exe_shell("sed -i '1a --zk_cluster='{} {}".format(
self.zk_endpoint, rtidb_flags))
exe_shell(
"sed -i 's/--recycle_bin_root_path=.*/--recycle_bin_root_path={}\/recycle/' {}"
.format(esc_tb_path, rtidb_flags))
exe_shell("echo '--log_level={}' >> {}".format(
conf.rtidb_log_info, rtidb_flags))
exe_shell("echo '--stream_close_wait_time_ms=10' >> {}".format(
rtidb_flags))
exe_shell(
"echo '--stream_bandwidth_limit=0' >> {}".format(rtidb_flags))
if not is_remote:
exe_shell(
"echo '--zk_root_path=/onebox' >> {}".format(rtidb_flags))
else:
exe_shell(
"echo '--zk_root_path=/remote' >> {}".format(rtidb_flags))
exe_shell(
"echo '--zk_keep_alive_check_interval=500000' >> {}".format(
rtidb_flags))
exe_shell("echo '--gc_safe_offset=0' >> {}".format(rtidb_flags))
exe_shell("echo '--binlog_sync_to_disk_interval=10' >> {}".format(
rtidb_flags))
exe_shell(
"echo '--binlog_sync_wait_time=10' >> {}".format(rtidb_flags))
exe_shell(
"echo '--zk_session_timeout=2000' >> {}".format(rtidb_flags))
exe_shell("echo '--make_snapshot_threshold_offset=0' >> {}".format(
rtidb_flags))
exe_shell("echo '--ssd_root_path={}/ssd_db/' >> {}".format(
tb_path, rtidb_flags))
exe_shell("echo '--hdd_root_path={}/hdd_db/' >> {}".format(
tb_path, rtidb_flags))
exe_shell(
"echo '--recycle_ssd_bin_root_path={}/ssd_recycle/' >> {}".
format(tb_path, rtidb_flags))
exe_shell(
"echo '--recycle_hdd_bin_root_path={}/hdd_recycle/' >> {}".
format(tb_path, rtidb_flags))
exe_shell("ulimit -c unlimited")
cmd = '{}/fedb --flagfile={}/conf/tablet.flags'.format(
test_path, tb_path)
infoLogger.info('start fedb: {}'.format(cmd))
args = shlex.split(cmd)
started = []
for _ in range(5):
rs = exe_shell('lsof -i:{}|grep -v "PID"'.format(
ep.split(':')[1]))
if 'fedb' not in rs:
time.sleep(2)
subprocess.Popen(
args,
stdout=open('{}/info.log'.format(tb_path), 'a'),
stderr=open('{}/warning.log'.format(tb_path), 'a'))
else:
started.append(True)
break
return started
def start(self, is_remote, *endpoints):
nsconfpath = os.getenv('nsconfpath')
i = 0
for ep in endpoints:
i += 1
if not is_remote:
ns_path = self.ns_edp_path[ep]
else:
ns_path = self.ns_edp_path[ep] + 'remote'
nameserver_flags = '{}/conf/nameserver.flags'.format(ns_path)
exe_shell('mkdir -p {}/conf'.format(ns_path))
exe_shell('touch {}'.format(nameserver_flags))
exe_shell("echo '--log_level={}' >> {}".format(conf.rtidb_log_info, nameserver_flags))
exe_shell("echo '--endpoint='{} >> {}".format(ep, nameserver_flags))
exe_shell("echo '--role=nameserver' >> {}".format(nameserver_flags))
exe_shell("echo '--zk_cluster='{} >> {}".format(self.zk_endpoint, nameserver_flags))
if not is_remote:
exe_shell("echo '--zk_root_path=/onebox' >> {}".format(nameserver_flags))
else:
exe_shell("echo '--zk_root_path=/remote' >> {}".format(nameserver_flags))
exe_shell("echo '--auto_failover=false' >> {}".format(nameserver_flags))
exe_shell("echo '--get_task_status_interval=100' >> {}".format(nameserver_flags))
exe_shell("echo '--name_server_task_pool_size=10' >> {}".format(nameserver_flags))
exe_shell("echo '--zk_keep_alive_check_interval=500000' >> {}".format(nameserver_flags))
exe_shell("echo '--tablet_offline_check_interval=10' >> {}".format(nameserver_flags))
exe_shell("echo '--tablet_heartbeat_timeout=0' >> {}".format(nameserver_flags))
exe_shell("echo '--request_timeout_ms=100000' >> {}".format(nameserver_flags))
exe_shell("echo '--name_server_task_concurrency=8' >> {}".format(nameserver_flags))
if is_remote:
exe_shell("echo '--get_replica_status_interval=600000' >> {}".format(nameserver_flags))
exe_shell("echo '--zk_session_timeout=2000' >> {}".format(nameserver_flags))
exe_shell("ulimit -c unlimited")
cmd = '{}/fedb --flagfile={}'.format(self.test_path, nameserver_flags)
infoLogger.info('start fedb: {}'.format(cmd))
args = shlex.split(cmd)
started = []
for _ in range(5):
rs = exe_shell('lsof -i:{}|grep -v "PID"'.format(ep.split(':')[1]))
if 'fedb' not in rs:
time.sleep(2)
subprocess.Popen(args,stdout=open('{}/info.log'.format(ns_path), 'a'),
stderr=open('{}/warning.log'.format(ns_path), 'a'))
else:
started.append(True)
break
return started
def test_loadtable_by_binlog_after_makesnapshots(self):
"""
多次makesnapshot后，可以通过binlog loadtable，且数据完整
:return:
"""
rs1 = self.create(self.leader, 't', self.tid, self.pid)
self.assertIn('Create table ok', rs1)
for i in range(0, 3):
# for multidimension test
self.multidimension_vk = {
'card': ('string:index', 'testkey1'),
'merchant': ('string:index', 'testvalue1'),
'amt': ('double', 1.1)
}
self.put(self.leader, self.tid, self.pid, 'testkey1',
self.now() - i, 'testvalue1')
rs2 = self.makesnapshot(self.leader, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs2)
for i in range(0, 3):
# for multidimension test
self.multidimension_vk = {
'card': ('string:index', 'testkey2'),
'merchant': ('string:index', 'testvalue2'),
'amt': ('double', 1.1)
}
self.put(self.leader, self.tid, self.pid, 'testkey2',
self.now() - i, 'testvalue2')
# 将table目录拷贝到新节点，删掉snapshot目录，保留binlog目录
self.cp_db(self.leaderpath, self.slave1path, self.tid, self.pid)
utils.exe_shell('rm -rf {}/db/{}_{}/snapshot/*'.format(
self.slave1path, self.tid, self.pid))
rs3 = self.loadtable(self.slave1, 't', self.tid, self.pid)
self.assertIn('LoadTable ok', rs3)
self.multidimension_scan_vk = {
'card': 'testkey1'
} # for multidimension test
self.assertIn(
'testvalue1',
self.scan(self.slave1, self.tid, self.pid, 'testkey1', self.now(),
1))
self.multidimension_scan_vk = {
'card': 'testkey2'
} # for multidimension test
self.assertIn(
'testvalue2',
self.scan(self.slave1, self.tid, self.pid, 'testkey2', self.now(),
1))
rs4 = self.makesnapshot(self.slave1, self.tid, self.pid)
self.assertIn('MakeSnapshot ok', rs4)
mf = self.get_manifest(self.slave1path, self.tid, self.pid)
self.assertEqual(mf['offset'], '6')
self.assertTrue(mf['name'])
self.assertEqual(mf['count'], '6')
table_status = self.get_table_status(self.leader, self.tid, self.pid)
self.assertEqual(
table_status[:6],
['6', 'kTableLeader', 'kTableNormal', 'true', '144000min', '0s'])
