def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(environment.binary_path('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(environment.zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
# a typical workstation will do 45-47k QPS, check we have more than 10k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 100000:
self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 100000:
self.fail('Cache QPS is too low: %u < 10000' % (cacheReads / 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
raise utils.TestError('invalid zk global state: ', v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
# a typical workstation will do 45-47k QPS, check we have more than 15k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 150000:
self.fail('QPS is too low: %u < 15000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 150000:
self.fail('Cache QPS is too low: %u < 15000' % (cacheReads / 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2'
% utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State']['Current'] != 'Connected':
raise utils.TestError('invalid zk global state: ',
v['ZkReader']['test_nj']['State']['Current'])
if v['ZkReader']['test_nj']['State']['DurationConnected'] < 9e9:
self.fail('not enough time in Connected state: %s' %
v['ZkReader']['test_nj']['State']['DurationConnected'])
# some checks on performance / stats
# a typical workstation will do 15k QPS, check we have more than 3k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 30000:
self.fail('QPS is too low: %u < 30000' % rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 30000:
self.fail('Cache QPS is too low: %u < 30000' % cacheReads / 10)
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0,
'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.topo_server().zkocc_port_base)
qpser.wait()
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(environment.topo_server().zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < MIN_QPS * 10:
self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < MIN_QPS * 10:
self.fail('Cache QPS is too low: %u < %u' % (cacheReads, MIN_QPS * 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def tearDownModule():
if utils.options.skip_teardown:
return
teardown_procs = [
tablet_62344.teardown_mysql(),
tablet_62044.teardown_mysql(),
tablet_41983.teardown_mysql(),
tablet_31981.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
if zkocc_server:
utils.zkocc_kill(zkocc_server)
utils.zk_teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
tablet_62344.remove_tree()
tablet_62044.remove_tree()
tablet_41983.remove_tree()
tablet_31981.remove_tree()
path = os.path.join(environment.vtdataroot, 'snapshot')
try:
shutil.rmtree(path)
except OSError as e:
logging.debug("removing snapshot %s: %s", path, str(e))
def tearDownModule():
if utils.options.skip_teardown:
return
teardown_procs = [
tablet_62344.teardown_mysql(),
tablet_62044.teardown_mysql(),
tablet_41983.teardown_mysql(),
tablet_31981.teardown_mysql(),
]
utils.wait_procs(teardown_procs, raise_on_error=False)
if zkocc_server:
utils.zkocc_kill(zkocc_server)
utils.zk_teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
tablet_62344.remove_tree()
tablet_62044.remove_tree()
tablet_41983.remove_tree()
tablet_31981.remove_tree()
path = os.path.join(environment.vtdataroot, 'snapshot')
try:
shutil.rmtree(path)
except OSError as e:
logging.debug("removing snapshot %s: %s", path, str(e))
def run_test_zkocc_qps():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2' % utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State']['Current'] != 'Connected':
raise utils.TestError('invalid zk global state: ', v['ZkReader']['test_nj']['State']['Current'])
if v['ZkReader']['test_nj']['State']['DurationConnected'] < 9e9:
raise utils.TestError('not enough time in Connected state', v['ZkReader']['test_nj']['State']['DurationConnected'])
# some checks on performance / stats
# a typical workstation will do 15k QPS, check we have more than 3k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 30000:
raise utils.TestError('QPS is too low: %u < 30000', rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 30000:
raise utils.TestError('Cache QPS is too low: %u < 30000', cacheReads / 10)
totalCacheReads = v['ZkReader']['total']['CacheReads']
if cacheReads != totalCacheReads:
raise utils.TestError('Rollup stats are wrong: %u != %u', cacheReads,
totalCacheReads)
if v['ZkReader']['UnknownCellErrors'] != 0:
raise utils.TestError('unexpected UnknownCellErrors', v['ZkReader']['UnknownCellErrors'])
utils.zkocc_kill(zkocc_14850)
def tearDown(self): utils.zkocc_kill(self.zkocc_server) utils.vtgate_kill(self.vtgate_zk) utils.vtgate_kill(self.vtgate_zkocc)
def test_zkocc(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
# FIXME(ryszard): This can be changed into a self.assertRaises.
try:
bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(level)
# get test
utils.prog_compile(['zkclient2'])
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True)
self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")
zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.assertDictContainsSubset({'Data': "Test data 1",
'Cached': True,
'Stale': False,},
zk_data)
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])
# getv test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % utils.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""")
zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
self.assertEqual(len(zk_data), 3)
for i, d in enumerate(zk_data):
self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
self.assertTrue(d['Cached'])
self.assertFalse(d['Stale'])
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])
# children test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj/vt' % utils.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 2
Cached = false
Stale = false
""")
# zk command tests
self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zk_data[0]['Stat']['MTime']),
_format_time(zk_data[1]['Stat']['MTime']),
_format_time(zk_data[2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/local/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False)
self.assertIn("zkocc: cannot resolve local cell", err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.kill_sub_process(querier)
logging.debug("Checking %s", filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
self.fail('unexpected stale state')
self.assertEqual(state, 2)
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(level)
def tearDown(self): utils.zkocc_kill(self.zkocc_server) utils.vttopo_kill(self.vttopo_server)
def run_test_zkocc():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(
extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u,localhost:%u" %
(bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.ERROR)
try:
bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
# get test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True)
if err != "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n":
raise utils.TestError('unexpected get output: ', err)
zkNode = zkocc_client.get("/zk/test_nj/zkocc1/data1")
if (zkNode['Data'] != "Test data 1" or \
zkNode['Stat']['NumChildren'] != 0 or \
zkNode['Stat']['Version'] != 0 or \
zkNode['Cached'] != True or \
zkNode['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.get output: ', zkNode)
# getv test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3'
% utils.zkocc_port_base,
trap_output=True)
if err != """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""":
raise utils.TestError('unexpected getV output: ', err)
zkNodes = zkocc_client.getv([
"/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2",
"/zk/test_nj/zkocc1/data3"
])
if (zkNodes['Nodes'][0]['Data'] != "Test data 1" or \
zkNodes['Nodes'][0]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][0]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][0]['Cached'] != True or \
zkNodes['Nodes'][0]['Stale'] != False or \
zkNodes['Nodes'][1]['Data'] != "Test data 2" or \
zkNodes['Nodes'][1]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][1]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][1]['Cached'] != True or \
zkNodes['Nodes'][1]['Stale'] != False or \
zkNodes['Nodes'][2]['Data'] != "Test data 3" or \
zkNodes['Nodes'][2]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][2]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][2]['Cached'] != True or \
zkNodes['Nodes'][2]['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.getv output: ', zkNodes)
# children test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' %
utils.zkocc_port_base,
trap_output=True)
if err != """Path = /zk/test_nj
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 4
Cached = false
Stale = false
""":
raise utils.TestError('unexpected children output: ', err)
# zk command tests
_check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
_check_zk_output(
"ls -l /zk/test_nj/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zkNodes['Nodes'][0]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][1]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True,
raise_on_error=False)
if "zkocc: cannot resolve local cell" not in err:
raise utils.TestError('unexpected get output, not local cell error: ',
err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg(
'/bin/bash -c "while true ; do ' + utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"'
% utils.zkocc_port_base,
stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(
utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u shutdown' %
(utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2))
time.sleep(3)
utils.run(
utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u start' %
(utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2))
time.sleep(3)
utils.kill_sub_process(querier)
utils.debug("Checking " + filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
raise utils.TestError('unexpected stale state')
if state != 2:
raise utils.TestError('unexpected ended stale state')
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
def tearDown(self):
utils.vtgate_kill(self.vtgate_zk)
if environment.topo_server_implementation == 'zookeeper':
self.topo.close()
utils.zkocc_kill(self.zkocc_server)
utils.vtgate_kill(self.vtgate_zkocc)
def test_zkocc(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = environment.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, environment.zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s' % str(e))
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
# FIXME(ryszard): This can be changed into a self.assertRaises.
try:
bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s' % str(e))
logging.getLogger().setLevel(level)
# get test
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % environment.zkocc_port_base, trap_output=True)
self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")
zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.assertDictContainsSubset({'Data': "Test data 1",
'Cached': True,
'Stale': False,},
zk_data)
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])
# getv test
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % environment.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""")
zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
self.assertEqual(len(zk_data), 3)
for i, d in enumerate(zk_data):
self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
self.assertTrue(d['Cached'])
self.assertFalse(d['Stale'])
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])
# children test
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode children /zk/test_nj/vt' % environment.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 2
Cached = false
Stale = false
""")
# zk command tests
self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zk_data[0]['Stat']['MTime']),
_format_time(zk_data[1]['Stat']['MTime']),
_format_time(zk_data[2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/local/vt/zkocc1/data1' % environment.zkocc_port_base, trap_output=True, raise_on_error=False)
self.assertIn("zkocc: cannot resolve local cell", err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % environment.zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(environment.binary_path('zkctl')+' -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (environment.zk_port_base, environment.zk_port_base+1, environment.zk_port_base+2))
time.sleep(3)
utils.run(environment.binary_path('zkctl')+' -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (environment.zk_port_base, environment.zk_port_base+1, environment.zk_port_base+2))
time.sleep(3)
utils.kill_sub_process(querier)
logging.debug("Checking %s", filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
self.fail('unexpected line: %s' % line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
self.fail('unexpected stale state')
self.assertEqual(state, 2)
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s', str(e))
logging.getLogger().setLevel(level)
def tearDown(self):
utils.vtgate_kill(self.vtgate_zk)
if environment.topo_server().flavor() == 'zookeeper':
self.topo.close()
utils.zkocc_kill(self.zkocc_server)
utils.vtgate_kill(self.vtgate_zkocc)
def tearDown(self):
utils.zkocc_kill(self.zkocc_server)
utils.vttopo_kill(self.vttopo_server)
def run_test_zkocc():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.ERROR)
try:
bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
# get test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' % utils.zkocc_port_base, trap_output=True)
if err != "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n":
raise utils.TestError('unexpected get output: ', err)
zkNode = zkocc_client.get("/zk/test_nj/zkocc1/data1")
if (zkNode['Data'] != "Test data 1" or \
zkNode['Stat']['NumChildren'] != 0 or \
zkNode['Stat']['Version'] != 0 or \
zkNode['Cached'] != True or \
zkNode['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.get output: ', zkNode)
# getv test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3' % utils.zkocc_port_base, trap_output=True)
if err != """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""":
raise utils.TestError('unexpected getV output: ', err)
zkNodes = zkocc_client.getv(["/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2", "/zk/test_nj/zkocc1/data3"])
if (zkNodes['Nodes'][0]['Data'] != "Test data 1" or \
zkNodes['Nodes'][0]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][0]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][0]['Cached'] != True or \
zkNodes['Nodes'][0]['Stale'] != False or \
zkNodes['Nodes'][1]['Data'] != "Test data 2" or \
zkNodes['Nodes'][1]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][1]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][1]['Cached'] != True or \
zkNodes['Nodes'][1]['Stale'] != False or \
zkNodes['Nodes'][2]['Data'] != "Test data 3" or \
zkNodes['Nodes'][2]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][2]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][2]['Cached'] != True or \
zkNodes['Nodes'][2]['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.getv output: ', zkNodes)
# children test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' % utils.zkocc_port_base, trap_output=True)
if err != """Path = /zk/test_nj
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 4
Cached = false
Stale = false
""":
raise utils.TestError('unexpected children output: ', err)
# zk command tests
_check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
_check_zk_output("ls -l /zk/test_nj/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zkNodes['Nodes'][0]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][1]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False)
if "zkocc: cannot resolve local cell" not in err:
raise utils.TestError('unexpected get output, not local cell error: ', err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.kill_sub_process(querier)
utils.debug("Checking " + filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
raise utils.TestError('unexpected stale state')
if state != 2:
raise utils.TestError('unexpected ended stale state')
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
