def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(environment.binary_path('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(environment.zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
# a typical workstation will do 45-47k QPS, check we have more than 10k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 100000:
self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 100000:
self.fail('Cache QPS is too low: %u < 10000' % (cacheReads / 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def setup_tablets():
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl('CreateKeyspace test_keyspace')
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('vt_test_keyspace')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
zkocc_server = utils.zkocc_start()
master_tablet.start_vttablet()
replica_tablet.start_vttablet()
utils.run_vtctl('SetReadWrite ' + master_tablet.tablet_alias)
utils.check_db_read_write(62344)
for t in [master_tablet, replica_tablet]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + master_tablet.tablet_alias, auto_log=True)
# reset counter so tests don't assert
tablet.Tablet.tablets_running = 0
setup_schema()
master_tablet.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
replica_tablet.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.topo_server().zkocc_port_base)
qpser.wait()
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(environment.topo_server().zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < MIN_QPS * 10:
self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < MIN_QPS * 10:
self.fail('Cache QPS is too low: %u < %u' % (cacheReads, MIN_QPS * 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2'
% utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State']['Current'] != 'Connected':
raise utils.TestError('invalid zk global state: ',
v['ZkReader']['test_nj']['State']['Current'])
if v['ZkReader']['test_nj']['State']['DurationConnected'] < 9e9:
self.fail('not enough time in Connected state: %s' %
v['ZkReader']['test_nj']['State']['DurationConnected'])
# some checks on performance / stats
# a typical workstation will do 15k QPS, check we have more than 3k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 30000:
self.fail('QPS is too low: %u < 30000' % rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 30000:
self.fail('Cache QPS is too low: %u < 30000' % cacheReads / 10)
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0,
'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def setUp(self):
utils.zk_wipe()
self.zkocc_server = utils.zkocc_start()
self.vttopo_server = utils.vttopo_start()
self.topo = zkocc.ZkOccConnection(
"localhost:%u" % utils.zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def run_test_zkocc_qps():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2' % utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State']['Current'] != 'Connected':
raise utils.TestError('invalid zk global state: ', v['ZkReader']['test_nj']['State']['Current'])
if v['ZkReader']['test_nj']['State']['DurationConnected'] < 9e9:
raise utils.TestError('not enough time in Connected state', v['ZkReader']['test_nj']['State']['DurationConnected'])
# some checks on performance / stats
# a typical workstation will do 15k QPS, check we have more than 3k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 30000:
raise utils.TestError('QPS is too low: %u < 30000', rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 30000:
raise utils.TestError('Cache QPS is too low: %u < 30000', cacheReads / 10)
totalCacheReads = v['ZkReader']['total']['CacheReads']
if cacheReads != totalCacheReads:
raise utils.TestError('Rollup stats are wrong: %u != %u', cacheReads,
totalCacheReads)
if v['ZkReader']['UnknownCellErrors'] != 0:
raise utils.TestError('unexpected UnknownCellErrors', v['ZkReader']['UnknownCellErrors'])
utils.zkocc_kill(zkocc_14850)
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
raise utils.TestError('invalid zk global state: ', v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
# a typical workstation will do 45-47k QPS, check we have more than 15k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 150000:
self.fail('QPS is too low: %u < 15000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 150000:
self.fail('Cache QPS is too low: %u < 15000' % (cacheReads / 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def setUp(self):
environment.topo_server_wipe()
self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
if environment.topo_server_implementation == 'zookeeper':
self.zkocc_server = utils.zkocc_start()
self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
self.topo = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def setUp(self):
utils.zk_wipe()
self.zkocc_server = utils.zkocc_start()
# the default topo implementation for vtgate is zookeeper
self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
self.topo = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def setUpModule():
try:
utils.zk_setup()
zkocc_server = utils.zkocc_start()
# start mysql instance external to the test
setup_procs = [
tablet_62344.init_mysql(),
tablet_62044.init_mysql(),
tablet_41983.init_mysql(),
tablet_31981.init_mysql(),
]
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
try:
utils.zk_setup()
zkocc_server = utils.zkocc_start()
# start mysql instance external to the test
setup_procs = [
tablet_62344.init_mysql(),
tablet_62044.init_mysql(),
tablet_41983.init_mysql(),
tablet_31981.init_mysql(),
]
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def test_restart(self):
zkocc_server = utils.zkocc_start()
shard_0_master.create_db('vt_test_keyspace')
proc1 = shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
proc2 = shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
time.sleep(2.0)
proc1.poll()
if proc1.returncode is None:
raise utils.TestError("proc1 still running")
shard_0_master.kill_vttablet()
utils.kill_sub_process(zkocc_server)
logging.debug("Done here")
def setup_tablets():
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl('CreateKeyspace %s' % TEST_KEYSPACE)
shard_0_master.init_tablet('master', keyspace=TEST_KEYSPACE, shard='0')
shard_0_replica.init_tablet('replica', keyspace=TEST_KEYSPACE, shard='0')
shard_1_master.init_tablet('master', keyspace=TEST_KEYSPACE, shard='1')
shard_1_replica.init_tablet('replica', keyspace=TEST_KEYSPACE, shard='1')
utils.run_vtctl('RebuildShardGraph %s/0' % TEST_KEYSPACE, auto_log=True)
utils.run_vtctl('RebuildShardGraph %s/1' % TEST_KEYSPACE, auto_log=True)
utils.validate_topology()
shard_0_master.create_db(shard_0_master.dbname)
shard_0_replica.create_db(shard_0_master.dbname)
shard_1_master.create_db(shard_0_master.dbname)
shard_1_replica.create_db(shard_0_master.dbname)
setup_schema()
utils.run_vtctl('RebuildKeyspaceGraph %s' % TEST_KEYSPACE, auto_log=True)
zkocc_server = utils.zkocc_start()
shard_0_master.start_vttablet()
shard_0_replica.start_vttablet()
shard_1_master.start_vttablet()
shard_1_replica.start_vttablet()
utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
utils.check_db_read_write(62344)
for t in [shard_0_master, shard_0_replica]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
for t in [shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/1 ' + shard_1_master.tablet_alias, auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base,
"test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc")
logging.debug(shard_0_master_addrs)
def run_test_restart():
zkocc_server = utils.zkocc_start()
shard_0_master.create_db('vt_test_keyspace')
proc1 = shard_0_master.start_vttablet(cert=cert_dir +
"/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
proc2 = shard_0_master.start_vttablet(cert=cert_dir +
"/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
time.sleep(2.0)
proc1.poll()
if proc1.returncode is None:
raise utils.TestError("proc1 still running")
shard_0_master.kill_vttablet()
utils.kill_sub_process(zkocc_server)
def test_restart(self):
zkocc_server = utils.zkocc_start()
shard_0_master.create_db('vt_test_keyspace')
proc1 = shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# Takes a bit longer for vttablet to serve the pid port
time.sleep(1.0)
proc2 = shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
time.sleep(1.0)
proc1.poll()
if proc1.returncode is None:
self.fail("proc1 still running")
shard_0_master.kill_vttablet()
utils.kill_sub_process(zkocc_server)
logging.debug("Done here")
def setup_tablets():
# Start up a master mysql and vttablet
logging.debug("Setting up tablets")
utils.run_vtctl('CreateKeyspace %s' % TEST_KEYSPACE)
shard_0_master.init_tablet('master', keyspace=TEST_KEYSPACE, shard='0')
shard_0_replica.init_tablet('replica', keyspace=TEST_KEYSPACE, shard='0')
shard_1_master.init_tablet('master', keyspace=TEST_KEYSPACE, shard='1')
shard_1_replica.init_tablet('replica', keyspace=TEST_KEYSPACE, shard='1')
utils.run_vtctl('RebuildShardGraph %s/0' % TEST_KEYSPACE, auto_log=True)
utils.run_vtctl('RebuildShardGraph %s/1' % TEST_KEYSPACE, auto_log=True)
utils.validate_topology()
shard_0_master.create_db(shard_0_master.dbname)
shard_0_replica.create_db(shard_0_master.dbname)
shard_1_master.create_db(shard_0_master.dbname)
shard_1_replica.create_db(shard_0_master.dbname)
utils.run_vtctl('RebuildKeyspaceGraph %s' % TEST_KEYSPACE, auto_log=True)
zkocc_server = utils.zkocc_start()
shard_0_master.start_vttablet()
shard_0_replica.start_vttablet()
shard_1_master.start_vttablet()
shard_1_replica.start_vttablet()
utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
utils.check_db_read_write(62344)
for t in [shard_0_master, shard_0_replica]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
for t in [shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/1 ' + shard_1_master.tablet_alias, auto_log=True)
setup_schema()
shard_0_master.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
shard_1_master.vquery("set vt_schema_reload_time=86400", path="test_keyspace/1")
shard_0_replica.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
shard_1_replica.vquery("set vt_schema_reload_time=86400", path="test_keyspace/1")
def test_restart(self):
zkocc_server = utils.zkocc_start()
shard_0_master.create_db('vt_test_keyspace')
proc1 = shard_0_master.start_vttablet(
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# Takes a bit longer for vttablet to serve the pid port
time.sleep(1.0)
proc2 = shard_0_master.start_vttablet(
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
time.sleep(1.0)
proc1.poll()
if proc1.returncode is None:
self.fail("proc1 still running")
shard_0_master.kill_vttablet()
utils.kill_sub_process(zkocc_server)
logging.debug("Done here")
def setUp(self):
utils.zk_wipe()
self.zkocc_server = utils.zkocc_start()
self.vttopo_server = utils.vttopo_start()
self.topo = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base, 'test_nj', 30)
self.topo.dial()
def run_test_zkocc():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.ERROR)
try:
bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
# get test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' % utils.zkocc_port_base, trap_output=True)
if err != "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n":
raise utils.TestError('unexpected get output: ', err)
zkNode = zkocc_client.get("/zk/test_nj/zkocc1/data1")
if (zkNode['Data'] != "Test data 1" or \
zkNode['Stat']['NumChildren'] != 0 or \
zkNode['Stat']['Version'] != 0 or \
zkNode['Cached'] != True or \
zkNode['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.get output: ', zkNode)
# getv test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3' % utils.zkocc_port_base, trap_output=True)
if err != """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""":
raise utils.TestError('unexpected getV output: ', err)
zkNodes = zkocc_client.getv(["/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2", "/zk/test_nj/zkocc1/data3"])
if (zkNodes['Nodes'][0]['Data'] != "Test data 1" or \
zkNodes['Nodes'][0]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][0]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][0]['Cached'] != True or \
zkNodes['Nodes'][0]['Stale'] != False or \
zkNodes['Nodes'][1]['Data'] != "Test data 2" or \
zkNodes['Nodes'][1]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][1]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][1]['Cached'] != True or \
zkNodes['Nodes'][1]['Stale'] != False or \
zkNodes['Nodes'][2]['Data'] != "Test data 3" or \
zkNodes['Nodes'][2]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][2]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][2]['Cached'] != True or \
zkNodes['Nodes'][2]['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.getv output: ', zkNodes)
# children test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' % utils.zkocc_port_base, trap_output=True)
if err != """Path = /zk/test_nj
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 4
Cached = false
Stale = false
""":
raise utils.TestError('unexpected children output: ', err)
# zk command tests
_check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
_check_zk_output("ls -l /zk/test_nj/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zkNodes['Nodes'][0]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][1]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False)
if "zkocc: cannot resolve local cell" not in err:
raise utils.TestError('unexpected get output, not local cell error: ', err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.kill_sub_process(querier)
utils.debug("Checking " + filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
raise utils.TestError('unexpected stale state')
if state != 2:
raise utils.TestError('unexpected ended stale state')
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
def test_zkocc(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
# FIXME(ryszard): This can be changed into a self.assertRaises.
try:
bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(level)
# get test
utils.prog_compile(['zkclient2'])
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True)
self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")
zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.assertDictContainsSubset({'Data': "Test data 1",
'Cached': True,
'Stale': False,},
zk_data)
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])
# getv test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % utils.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""")
zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
self.assertEqual(len(zk_data), 3)
for i, d in enumerate(zk_data):
self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
self.assertTrue(d['Cached'])
self.assertFalse(d['Stale'])
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])
# children test
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj/vt' % utils.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 2
Cached = false
Stale = false
""")
# zk command tests
self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zk_data[0]['Stat']['MTime']),
_format_time(zk_data[1]['Stat']['MTime']),
_format_time(zk_data[2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/local/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False)
self.assertIn("zkocc: cannot resolve local cell", err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
time.sleep(3)
utils.kill_sub_process(querier)
logging.debug("Checking %s", filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
self.fail('unexpected stale state')
self.assertEqual(state, 2)
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(level)
def test_sharding(self):
shard_0_master.init_tablet( 'master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet( 'master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets, wait for them to start
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl(['ApplySchema',
'-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias])
utils.run_vtctl(['ApplySchema',
'-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias])
if environment.topo_server_implementation == 'zookeeper':
# start zkocc, we'll use it later, indirectly with the vtdb-zkocc driver
zkocc_server = utils.zkocc_start()
# start vtgate, we'll use it later
vtgate_server, vtgate_port = utils.vtgate_start()
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' + shard_0_master.tablet_alias, auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/80- ' + shard_1_master.tablet_alias, auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl(['ApplySchemaShard',
'-simple',
'-sql=' + create_vt_select_test_reverse.replace("\n", ""),
'test_keyspace/80-'])
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
shard_0_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True)
shard_1_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(3803, "/test_nj/test_keyspace/master", "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')", bindvars='{"keyspace_id": 2}', driver="vtdb", verbose=True)
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery('vt_test_keyspace', "select id, msg from vt_select_test order by id")
self.assertEqual(rows, ((1, 'test 1'), (2, 'test 2'), ),
'wrong mysql_query output: %s' % str(rows))
utils.pause("After db writes")
# now use various topo servers and streaming or both for the same query
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-streaming")
if environment.topo_server_implementation == 'zookeeper':
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-zk")
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-zk-streaming")
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-zkocc")
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-zkocc-streaming")
# make sure the schema checking works
self._check_rows_schema_diff("vtdb")
if environment.topo_server_implementation == 'zookeeper':
self._check_rows_schema_diff("vtdb-zk")
self._check_rows_schema_diff("vtdb-zkocc")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl('ValidateSchemaShard test_keyspace/-80')
utils.run_vtctl('ValidateSchemaShard test_keyspace/80-')
out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace',
trap_output=True, raise_on_error=False)
if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \
"test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err:
self.fail('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl('ValidateVersionShard test_keyspace/-80', auto_log=True)
utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True)
# show and validate permissions
utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True)
utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80', auto_log=True)
utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace', auto_log=True)
if environment.topo_server_implementation == 'zookeeper':
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl('ExportZknsForKeyspace test_keyspace')
out, err = utils.run(environment.binary_argstr('zk')+' ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']:
expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
if expected not in lines:
self.fail('missing zkns part:\n%s\nin:%s' %(expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_keyspaces(vtgate_client)
shard_0_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.-80.master:_vtocc")
if len(shard_0_master_addrs) != 1:
self.fail('topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# connect to shard -80
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0],
shard_0_master_addrs[0][1]),
"", "test_keyspace", "-80", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [(1, 'test 1'), (2, 'test 2'), ],
'wrong conn._execute output: %s' % str(results))
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.80-.master:_vtocc")
conn = tablet3.TabletConnection("%s:%u" % (shard_1_master_addrs[0][0],
shard_1_master_addrs[0][1]),
"", "test_keyspace", "80-", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [(10, 'test 10'), ],
'wrong conn._execute output: %s' % str(results))
vtgate_client.close()
# try to connect with bad shard
try:
conn = tablet3.TabletConnection("localhost:%u" % shard_0_master.port,
"", "test_keyspace", "-90", 10.0)
conn.dial()
self.fail('expected an exception')
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(e):
self.fail('unexpected exception: ' + str(e))
utils.vtgate_kill(vtgate_server)
if environment.topo_server_implementation == 'zookeeper':
utils.kill_sub_process(zkocc_server)
tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica])
def test_secure(self):
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert': cert_dir + "/client-cert.pem",
'ssl-key': cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base,
"test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
if shard_0_master_addrs[0][2] != True:
self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted')
logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
self.fail('topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"", "test_keyspace", "0", 10.0)
conn.dial()
self.fail("No exception raised to secure port")
except dbexceptions.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"", "test_keyspace", "0", 5.0, encrypted=True)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s", str(e))
# start a vtgate to connect to that tablet
gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
tablet_bson_encrypted=True,
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# try to connect to vtgate with regular client
timeout = 2.0
try:
conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),],
timeout)
self.fail("No exception raised to VTGate secure port")
except dbexceptions.OperationalError as e:
exception_type = e.args[2]
exception_msg = str(e.args[2][0][0])
self.assertIsInstance(exception_type, dbexceptions.FatalError,
"unexpected exception type")
if not exception_msg.startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception message: %s" % exception_msg)
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to vtgate with encrypted port
conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),],
timeout, encrypted=True)
(results, rowcount, lastrowid, fields) = conn._execute(
"select 1 from dual",
{},
"test_keyspace",
"master",
keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),])
self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(gate_port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a vtgate secure connection, see what exception we get
try:
conn._execute("select sleep(4) from dual",
{},
"test_keyspace",
"master",
keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),])
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s", str(e))
conn.close()
utils.vtgate_kill(gate_proc)
# kill everything
utils.kill_sub_process(zkocc_server)
def run_test_secure():
utils.run_vtctl('CreateKeyspace test_keyspace')
shard_0_master.init_tablet('master', 'test_keyspace', '0')
shard_0_slave.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl('RebuildShardGraph test_keyspace/0', auto_log=True)
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
zkocc_server = utils.zkocc_start()
# create databases so vttablet can start behaving normally
shard_0_master.create_db('vt_test_keyspace')
shard_0_slave.create_db('vt_test_keyspace')
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': 2048,
'ssl_ca': cert_dir + "/ca-cert.pem",
'ssl_cert': cert_dir + "/client-cert.pem",
'ssl_key': cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base,
"test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
raise utils.TestError('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
if shard_0_master_addrs[0][2] != True:
raise utils.TestError('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted')
utils.debug("shard 0 master addrs: %s" % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
raise utils.TestError('topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "0", 10.0)
conn.dial()
raise utils.TestError("No exception raised to secure port")
except tablet3.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
raise utils.TestError("Unexpected exception: %s" % str(e))
# connect to encrypted port
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "0", 5.0, encrypted=True)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {})
if (len(results) != 1 or \
results[0][0] != 1):
print "conn._execute returned:", results
raise utils.TestError('wrong conn._execute output')
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
raise utils.TestError("No timeout exception")
except tablet3.TimeoutError as e:
utils.debug("Got the right exception for SSL timeout: %s" % str(e))
# kill everything
utils.kill_sub_process(zkocc_server)
shard_0_master.kill_vttablet()
shard_0_slave.kill_vttablet()
def run_test_secure():
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': 2048,
'ssl_ca': cert_dir + "/ca-cert.pem",
'ssl_cert': cert_dir + "/client-cert.pem",
'ssl_key': cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
shard_0_master.tablet_alias,
auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
raise utils.TestError(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
if shard_0_master_addrs[0][2] != True:
raise utils.TestError(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted'
)
utils.debug("shard 0 master addrs: %s" % " ".join(
["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
raise utils.TestError(
'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
% (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "0", 10.0)
conn.dial()
raise utils.TestError("No exception raised to secure port")
except tablet3.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
raise utils.TestError("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConns"]
if sconn != 0:
raise utils.TestError("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace",
"0",
5.0,
encrypted=True)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select 1 from dual", {})
if (len(results) != 1 or \
results[0][0] != 1):
print "conn._execute returned:", results
raise utils.TestError('wrong conn._execute output')
sconn = utils.get_vars(shard_0_master.port)["SecureConns"]
if sconn != 1:
raise utils.TestError("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
raise utils.TestError("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
raise utils.TestError("No timeout exception")
except tablet3.TimeoutError as e:
utils.debug("Got the right exception for SSL timeout: %s" % str(e))
# kill everything
utils.kill_sub_process(zkocc_server)
def test_sharding(self):
shard_0_master.init_tablet('master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet('master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl(
'RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*',
auto_log=True)
utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*',
auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases
shard_0_master.create_db('vt_test_keyspace')
shard_0_replica.create_db('vt_test_keyspace')
shard_1_master.create_db('vt_test_keyspace')
shard_1_replica.create_db('vt_test_keyspace')
# start the tablets
shard_0_master.start_vttablet()
shard_0_replica.start_vttablet()
shard_1_master.start_vttablet()
shard_1_replica.start_vttablet()
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias
])
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias
])
# start zkocc, we'll use it later
zkocc_server = utils.zkocc_start()
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' +
shard_0_master.tablet_alias,
auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/80- ' +
shard_1_master.tablet_alias,
auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl([
'ApplySchemaShard', '-simple',
'-sql=' + create_vt_select_test_reverse.replace("\n", ""),
'test_keyspace/80-'
])
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
shard_0_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (1, 'test 1')",
write=True)
shard_1_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (10, 'test 10')",
write=True)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(
3803,
"/test_nj/test_keyspace/master",
"insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')",
bindvars='{"keyspace_id": 2}',
driver="vtdb",
verbose=True)
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery(
'vt_test_keyspace',
"select id, msg from vt_select_test order by id")
self.assertEqual(rows, (
(1, 'test 1'),
(2, 'test 2'),
), 'wrong mysql_query output: %s' % str(rows))
utils.pause("After db writes")
# now use zkocc or streaming or both for the same query
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-streaming")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc-streaming")
# make sure the schema checking works
self._check_rows_schema_diff("vtdb-zkocc")
self._check_rows_schema_diff("vtdb")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl('ValidateSchemaShard test_keyspace/-80')
utils.run_vtctl('ValidateSchemaShard test_keyspace/80-')
out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace',
trap_output=True,
raise_on_error=False)
if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \
"test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err:
self.fail('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl('ValidateVersionShard test_keyspace/-80',
auto_log=True)
utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True)
# show and validate permissions
utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True)
utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80',
auto_log=True)
utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace',
auto_log=True)
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl('ExportZknsForKeyspace test_keyspace')
out, err = utils.run(utils.vtroot +
'/bin/zk ls -R /zk/test_nj/zk?s/vt/test_keysp*',
trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']:
expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
if expected not in lines:
self.fail('missing zkns part:\n%s\nin:%s' %
(expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.-80.master:_vtocc")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# connect to shard -80
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "-80", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute(
"select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [
(1, 'test 1'),
(2, 'test 2'),
], 'wrong conn._execute output: %s' % str(results))
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.80-.master:_vtocc")
conn = tablet3.TabletConnection(
"%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]),
"test_keyspace", "80-", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute(
"select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [
(10, 'test 10'),
], 'wrong conn._execute output: %s' % str(results))
# try to connect with bad shard
try:
conn = tablet3.TabletConnection(
"localhost:%u" % shard_0_master.port, "test_keyspace", "-90",
10.0)
conn.dial()
self.fail('expected an exception')
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(
e):
self.fail('unexpected exception: ' + str(e))
utils.kill_sub_process(zkocc_server)
shard_0_master.kill_vttablet()
shard_0_replica.kill_vttablet()
shard_1_master.kill_vttablet()
shard_1_replica.kill_vttablet()
def test_zkocc(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = environment.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, environment.zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s' % str(e))
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
# FIXME(ryszard): This can be changed into a self.assertRaises.
try:
bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s' % str(e))
logging.getLogger().setLevel(level)
# get test
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % environment.zkocc_port_base, trap_output=True)
self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")
zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.assertDictContainsSubset({'Data': "Test data 1",
'Cached': True,
'Stale': False,},
zk_data)
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])
# getv test
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % environment.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""")
zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
self.assertEqual(len(zk_data), 3)
for i, d in enumerate(zk_data):
self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
self.assertTrue(d['Cached'])
self.assertFalse(d['Stale'])
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])
# children test
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u -mode children /zk/test_nj/vt' % environment.zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 2
Cached = false
Stale = false
""")
# zk command tests
self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zk_data[0]['Stat']['MTime']),
_format_time(zk_data[1]['Stat']['MTime']),
_format_time(zk_data[2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(environment.binary_path('zkclient2')+' -server localhost:%u /zk/local/vt/zkocc1/data1' % environment.zkocc_port_base, trap_output=True, raise_on_error=False)
self.assertIn("zkocc: cannot resolve local cell", err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+environment.binary_path('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % environment.zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(environment.binary_path('zkctl')+' -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (environment.zk_port_base, environment.zk_port_base+1, environment.zk_port_base+2))
time.sleep(3)
utils.run(environment.binary_path('zkctl')+' -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (environment.zk_port_base, environment.zk_port_base+1, environment.zk_port_base+2))
time.sleep(3)
utils.kill_sub_process(querier)
logging.debug("Checking %s", filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
self.fail('unexpected line: %s' % line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
self.fail('unexpected stale state')
self.assertEqual(state, 2)
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s', str(e))
logging.getLogger().setLevel(level)
def test_secure(self):
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert':
cert_dir + "/client-cert.pem",
'ssl-key':
cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
shard_0_master.tablet_alias,
auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % environment.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
if shard_0_master_addrs[0][2] != True:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted'
)
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
self.fail(
'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
% (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection(
"%s:%u" %
(shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "",
"test_keyspace", "0", 10.0)
conn.dial()
self.fail("No exception raised to secure port")
except dbexceptions.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"",
"test_keyspace",
"0",
5.0,
encrypted=True)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
# start a vtgate to connect to that tablet
gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
tablet_bson_encrypted=True,
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# try to connect to vtgate with regular client
timeout = 2.0
try:
conn = vtgatev2.connect([
"localhost:%s" % (gate_secure_port),
], timeout)
self.fail("No exception raised to VTGate secure port")
except dbexceptions.OperationalError as e:
exception_type = e.args[2]
exception_msg = str(e.args[2][0][0])
self.assertIsInstance(exception_type, dbexceptions.FatalError,
"unexpected exception type")
if not exception_msg.startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception message: %s" % exception_msg)
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to vtgate with encrypted port
conn = vtgatev2.connect([
"localhost:%s" % (gate_secure_port),
],
timeout,
encrypted=True)
(results, rowcount, lastrowid, fields) = conn._execute(
"select 1 from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(gate_port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a vtgate secure connection, see what exception we get
try:
conn._execute("select sleep(4) from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
conn.close()
utils.vtgate_kill(gate_proc)
# kill everything
utils.kill_sub_process(zkocc_server)
def test_secure(self):
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert': cert_dir + "/client-cert.pem",
'ssl-key': cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base,
"test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
raise utils.TestError('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
if shard_0_master_addrs[0][2] != True:
raise utils.TestError('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted')
logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
raise utils.TestError('topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "0", 10.0)
conn.dial()
raise utils.TestError("No exception raised to secure port")
except tablet3.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
raise utils.TestError("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
raise utils.TestError("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "0", 5.0, encrypted=True)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
raise utils.TestError("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
raise utils.TestError("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
raise utils.TestError("No timeout exception")
except tablet3.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s", str(e))
# kill everything
utils.kill_sub_process(zkocc_server)
def test_sharding(self):
shard_0_master.init_tablet("master", "test_keyspace", "-80")
shard_0_replica.init_tablet("replica", "test_keyspace", "-80")
shard_1_master.init_tablet("master", "test_keyspace", "80-")
shard_1_replica.init_tablet("replica", "test_keyspace", "80-")
utils.run_vtctl("RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*", auto_log=True)
utils.run_vtctl("RebuildKeyspaceGraph /zk/global/vt/keyspaces/*", auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases
shard_0_master.create_db("vt_test_keyspace")
shard_0_replica.create_db("vt_test_keyspace")
shard_1_master.create_db("vt_test_keyspace")
shard_1_replica.create_db("vt_test_keyspace")
# start the tablets
shard_0_master.start_vttablet()
shard_0_replica.start_vttablet()
shard_1_master.start_vttablet()
shard_1_replica.start_vttablet()
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl(
[
"ApplySchema",
"-stop-replication",
"-sql=" + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias,
]
)
utils.run_vtctl(
[
"ApplySchema",
"-stop-replication",
"-sql=" + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias,
]
)
# start zkocc, we'll use it later
zkocc_server = utils.zkocc_start()
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl("ReparentShard -force test_keyspace/-80 " + shard_0_master.tablet_alias, auto_log=True)
utils.run_vtctl("ReparentShard -force test_keyspace/80- " + shard_1_master.tablet_alias, auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl(
[
"ApplySchemaShard",
"-simple",
"-sql=" + create_vt_select_test_reverse.replace("\n", ""),
"test_keyspace/80-",
]
)
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl("SetReadWrite " + shard_0_master.tablet_alias)
utils.run_vtctl("SetReadWrite " + shard_1_master.tablet_alias)
shard_0_master.mquery(
"vt_test_keyspace", "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True
)
shard_1_master.mquery(
"vt_test_keyspace", "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True
)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(
3803,
"/test_nj/test_keyspace/master",
"insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')",
bindvars='{"keyspace_id": 2}',
driver="vtdb",
verbose=True,
)
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery("vt_test_keyspace", "select id, msg from vt_select_test order by id")
self.assertEqual(rows, ((1, "test 1"), (2, "test 2")), "wrong mysql_query output: %s" % str(rows))
utils.pause("After db writes")
# now use zkocc or streaming or both for the same query
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zkocc")
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-streaming")
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zkocc-streaming")
# make sure the schema checking works
self._check_rows_schema_diff("vtdb-zkocc")
self._check_rows_schema_diff("vtdb")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl("ValidateSchemaShard test_keyspace/-80")
utils.run_vtctl("ValidateSchemaShard test_keyspace/80-")
out, err = utils.run_vtctl("ValidateSchemaKeyspace test_keyspace", trap_output=True, raise_on_error=False)
if (
"test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE"
not in err
or "test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE"
not in err
):
self.fail("wrong ValidateSchemaKeyspace output: " + err)
# validate versions
utils.run_vtctl("ValidateVersionShard test_keyspace/-80", auto_log=True)
utils.run_vtctl("ValidateVersionKeyspace test_keyspace", auto_log=True)
# show and validate permissions
utils.run_vtctl("GetPermissions test_nj-0000062344", auto_log=True)
utils.run_vtctl("ValidatePermissionsShard test_keyspace/-80", auto_log=True)
utils.run_vtctl("ValidatePermissionsKeyspace test_keyspace", auto_log=True)
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl("ExportZknsForKeyspace test_keyspace")
out, err = utils.run(utils.vtroot + "/bin/zk ls -R /zk/test_nj/zk?s/vt/test_keysp*", trap_output=True)
lines = out.splitlines()
for base in ["-80", "80-"]:
for db_type in ["master", "replica"]:
for sub_path in ["", ".vdns", "/0", "/_vtocc.vdns"]:
expected = "/zk/test_nj/zkns/vt/test_keyspace/" + base + "/" + db_type + sub_path
if expected not in lines:
self.fail("missing zkns part:\n%s\nin:%s" % (expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
zkocc_client = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.-80.master:_vtocc")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s'
% " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs])
)
logging.debug(
"shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs])
)
# connect to shard -80
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "test_keyspace", "-80", 10.0
)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [(1, "test 1"), (2, "test 2")], "wrong conn._execute output: %s" % str(results))
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.80-.master:_vtocc")
conn = tablet3.TabletConnection(
"%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]), "test_keyspace", "80-", 10.0
)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [(10, "test 10")], "wrong conn._execute output: %s" % str(results))
# try to connect with bad shard
try:
conn = tablet3.TabletConnection("localhost:%u" % shard_0_master.port, "test_keyspace", "-90", 10.0)
conn.dial()
self.fail("expected an exception")
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(e):
self.fail("unexpected exception: " + str(e))
utils.kill_sub_process(zkocc_server)
shard_0_master.kill_vttablet()
shard_0_replica.kill_vttablet()
shard_1_master.kill_vttablet()
shard_1_replica.kill_vttablet()
def run_test_zkocc():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(
extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u,localhost:%u" %
(bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.ERROR)
try:
bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
# get test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True)
if err != "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n":
raise utils.TestError('unexpected get output: ', err)
zkNode = zkocc_client.get("/zk/test_nj/zkocc1/data1")
if (zkNode['Data'] != "Test data 1" or \
zkNode['Stat']['NumChildren'] != 0 or \
zkNode['Stat']['Version'] != 0 or \
zkNode['Cached'] != True or \
zkNode['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.get output: ', zkNode)
# getv test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3'
% utils.zkocc_port_base,
trap_output=True)
if err != """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""":
raise utils.TestError('unexpected getV output: ', err)
zkNodes = zkocc_client.getv([
"/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2",
"/zk/test_nj/zkocc1/data3"
])
if (zkNodes['Nodes'][0]['Data'] != "Test data 1" or \
zkNodes['Nodes'][0]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][0]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][0]['Cached'] != True or \
zkNodes['Nodes'][0]['Stale'] != False or \
zkNodes['Nodes'][1]['Data'] != "Test data 2" or \
zkNodes['Nodes'][1]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][1]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][1]['Cached'] != True or \
zkNodes['Nodes'][1]['Stale'] != False or \
zkNodes['Nodes'][2]['Data'] != "Test data 3" or \
zkNodes['Nodes'][2]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][2]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][2]['Cached'] != True or \
zkNodes['Nodes'][2]['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.getv output: ', zkNodes)
# children test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' %
utils.zkocc_port_base,
trap_output=True)
if err != """Path = /zk/test_nj
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 4
Cached = false
Stale = false
""":
raise utils.TestError('unexpected children output: ', err)
# zk command tests
_check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
_check_zk_output(
"ls -l /zk/test_nj/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zkNodes['Nodes'][0]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][1]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True,
raise_on_error=False)
if "zkocc: cannot resolve local cell" not in err:
raise utils.TestError('unexpected get output, not local cell error: ',
err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg(
'/bin/bash -c "while true ; do ' + utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"'
% utils.zkocc_port_base,
stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(
utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u shutdown' %
(utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2))
time.sleep(3)
utils.run(
utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u start' %
(utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2))
time.sleep(3)
utils.kill_sub_process(querier)
utils.debug("Checking " + filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
raise utils.TestError('unexpected stale state')
if state != 2:
raise utils.TestError('unexpected ended stale state')
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)