def test_get_srv_keyspace_names(self):
utils.run_vtctl('CreateKeyspace test_keyspace1')
utils.run_vtctl('CreateKeyspace test_keyspace2')
t1 = tablet.Tablet(tablet_uid=1, cell="nj")
t1.init_tablet("master", "test_keyspace1", "0")
t1.update_addrs()
t2 = tablet.Tablet(tablet_uid=2, cell="nj")
t2.init_tablet("master", "test_keyspace2", "0")
t2.update_addrs()
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'], auto_log=True)
# vtgate API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zk_port, trap_output=True)
self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
"KeyspaceNames[1] = test_keyspace2\n")
if environment.topo_server().flavor() == 'zookeeper':
self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"])
# zkocc API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % environment.topo_server().zkocc_port_base, trap_output=True)
self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
"KeyspaceNames[1] = test_keyspace2\n")
# vtgate zkocc API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zkocc_port, trap_output=True)
self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
"KeyspaceNames[1] = test_keyspace2\n")
def setUp(self):
environment.topo_server_wipe()
if environment.topo_server_implementation == 'zookeeper':
utils.run(
environment.binary_argstr('zk') +
' touch -p /zk/test_nj/vt/zkocc1')
utils.run(
environment.binary_argstr('zk') +
' touch -p /zk/test_nj/vt/zkocc2')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot,
delete=False)
filename1 = fd.name
fd.write("Test data 1")
fd.close()
utils.run(
environment.binary_argstr('zk') + ' cp ' + filename1 +
' /zk/test_nj/vt/zkocc1/data1')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot,
delete=False)
filename2 = fd.name
fd.write("Test data 2")
fd.close()
utils.run(
environment.binary_argstr('zk') + ' cp ' + filename2 +
' /zk/test_nj/vt/zkocc1/data2')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot,
delete=False)
filename3 = fd.name
fd.write("Test data 3")
fd.close()
utils.run(
environment.binary_argstr('zk') + ' cp ' + filename3 +
' /zk/test_nj/vt/zkocc1/data3')
def _check_zk_output(self, cmd, expected):
# directly for sanity
out, err = utils.run(environment.binary_argstr('zk')+' ' + cmd, trap_output=True)
self.assertEqualNormalized(out, expected, 'unexpected direct zk output')
# using zkocc
out, err = utils.run(environment.binary_argstr('zk')+' --zk.zkocc-addr=localhost:%u %s' % (environment.topo_server().zkocc_port_base, cmd), trap_output=True)
self.assertEqualNormalized(out, expected, 'unexpected zk zkocc output')
logging.debug("Matched: %s", out)
def test_get_srv_keyspace(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
utils.run_vtctl(
'UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s -vts-port %s %s'
% (t.mysql_port, t.port + 500, t.tablet_alias))
self.rebuild()
# vtgate zk API test
out, err = utils.run(
environment.binary_argstr('zkclient2') +
' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace'
% self.vtgate_zk_port,
trap_output=True)
self.assertEqual(
err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" +
"Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" +
"Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n",
"Got wrong content: %s" % err)
if environment.topo_server_implementation == 'zookeeper':
reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace")
self.assertEqual(reply['TabletTypes'], ['master'])
# zkocc API test
out, err = utils.run(
environment.binary_argstr('zkclient2') +
' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace'
% environment.zkocc_port_base,
trap_output=True)
self.assertEqual(
err,
"Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" +
"Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" +
"Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n",
"Got wrong content: %s" % err)
# vtgate zkocc API test
out, err = utils.run(
environment.binary_argstr('zkclient2') +
' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace'
% self.vtgate_zkocc_port,
trap_output=True)
self.assertEqual(
err,
"Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" +
"Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" +
"Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n",
"Got wrong content: %s" % err)
def create_customrules(self, filename):
with open(filename, "w") as f:
f.write("""[{
"Name": "r1",
"Description": "disallow bindvar 'asdfg'",
"BindVarConds":[{
"Name": "asdfg",
"OnAbsent": false,
"Operator": "NOOP"
}]
}]""")
if self.env == "vttablet":
if environment.topo_server().flavor() == 'zookeeper':
utils.run(environment.binary_argstr('zk') + ' touch -p /zk/test_ca/config/customrules/testrules')
utils.run(environment.binary_argstr('zk') + ' cp ' + filename + ' /zk/test_ca/config/customrules/testrules')
def restore_customrules(self):
customrules = os.path.join(environment.tmproot, 'customrules.json')
self.create_customrules(customrules)
if environment.topo_server().flavor() == 'zookeeper':
utils.run(
environment.binary_argstr('zk') + ' cp ' + customrules +
' /zk/test_ca/config/customrules/testrules')
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.topo_server().zkocc_port_base)
qpser.wait()
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(environment.topo_server().zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < MIN_QPS * 10:
self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < MIN_QPS * 10:
self.fail('Cache QPS is too low: %u < %u' % (cacheReads, MIN_QPS * 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def test_regular_operation(self):
# Use a dedicated worker to run all vtworker commands.
worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
['--cell', 'test_nj'],
auto_log=True)
vtworker_endpoint = 'localhost:' + str(worker_rpc_port)
automation_server_proc, automation_server_port = (
utils.run_automation_server())
source_shard_list = '0'
dest_shard_list = '-80,80-'
_, vtctld_endpoint = utils.vtctld.rpc_endpoint()
utils.run(
environment.binary_argstr('automation_client') +
' --server localhost:' + str(automation_server_port) +
' --task HorizontalReshardingTask' +
' --param keyspace=' + self.KEYSPACE +
' --param source_shard_list=' + source_shard_list +
' --param dest_shard_list=' + dest_shard_list +
' --param vtctld_endpoint=' + vtctld_endpoint +
' --param vtworker_endpoint=' + vtworker_endpoint +
' --param min_healthy_rdonly_endpoints=1')
self.verify()
utils.kill_sub_process(automation_server_proc, soft=True)
utils.kill_sub_process(worker_proc, soft=True)
def test_regular_operation(self):
# Use a dedicated worker to run all vtworker commands.
worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
['--cell', 'test_nj'], auto_log=True)
vtworker_endpoint = 'localhost:' + str(worker_rpc_port)
automation_server_proc, automation_server_port = (
utils.run_automation_server())
keyspace = 'test_keyspace'
source_shard_list = '0'
dest_shard_list = '-80,80-'
_, vtctld_endpoint = utils.vtctld.rpc_endpoint()
utils.run(
environment.binary_argstr('automation_client') +
' --server localhost:' + str(automation_server_port) +
' --task HorizontalReshardingTask' + ' --param keyspace=' +
keyspace + ' --param source_shard_list=' + source_shard_list +
' --param dest_shard_list=' + dest_shard_list +
' --param vtctld_endpoint=' + vtctld_endpoint +
' --param vtworker_endpoint=' + vtworker_endpoint)
self.assert_shard_data_equal(0, worker.shard_master,
worker.shard_0_tablets.replica)
self.assert_shard_data_equal(1, worker.shard_master,
worker.shard_1_tablets.replica)
utils.kill_sub_process(automation_server_proc, soft=True)
utils.kill_sub_process(worker_proc, soft=True)
def test_regular_operation(self):
# Use a dedicated worker to run all vtworker commands.
worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
['--cell', 'test_nj'], auto_log=True)
vtworker_endpoint = 'localhost:' + str(worker_rpc_port)
automation_server_proc, automation_server_port = (
utils.run_automation_server())
source_shard_list = '0'
dest_shard_list = '-80,80-'
_, vtctld_endpoint = utils.vtctld.rpc_endpoint()
utils.run(
environment.binary_argstr('automation_client') +
' --server localhost:' + str(automation_server_port) +
' --task HorizontalReshardingTask' + ' --param keyspace=' +
self.KEYSPACE + ' --param source_shard_list=' + source_shard_list +
' --param dest_shard_list=' + dest_shard_list +
' --param vtctld_endpoint=' + vtctld_endpoint +
' --param vtworker_endpoint=' + vtworker_endpoint +
' --param min_healthy_rdonly_tablets=1')
self.verify()
utils.kill_sub_process(automation_server_proc, soft=True)
utils.kill_sub_process(worker_proc, soft=True)
def test_regular_operation(self):
# Use a dedicated worker to run all vtworker commands.
worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
['--cell', 'test_nj'],
auto_log=True)
vtworker_endpoint = "localhost:" + str(worker_rpc_port)
automation_server_proc, automation_server_port = utils.run_automation_server()
keyspace = 'test_keyspace'
source_shard_list = '0'
dest_shard_list = '-80,80-'
_, vtctld_endpoint = utils.vtctld.rpc_endpoint()
utils.run(environment.binary_argstr('automation_client') +
' --server localhost:' + str(automation_server_port) +
' --task HorizontalReshardingTask' +
' --param keyspace=' + keyspace +
' --param source_shard_list=' + source_shard_list +
' --param dest_shard_list=' + dest_shard_list +
' --param source_shard_rdonly_list=' +
worker.shard_rdonly1.tablet_alias +
' --param dest_shard_rdonly_list=' +
worker.shard_0_rdonly1.tablet_alias + ',' +
worker.shard_1_rdonly1.tablet_alias +
' --param vtctld_endpoint=' + vtctld_endpoint +
' --param vtworker_endpoint=' + vtworker_endpoint)
self.assert_shard_data_equal(0, worker.shard_master,
worker.shard_0_tablets.replica)
self.assert_shard_data_equal(1, worker.shard_master,
worker.shard_1_tablets.replica)
utils.kill_sub_process(automation_server_proc, soft=True)
utils.kill_sub_process(worker_proc, soft=True)
def test_vtgate_qps(self):
# create the topology
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# start vtgate and the qps-er
vtgate_proc, vtgate_port = utils.vtgate_start()
qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the vtgate vars, make sure we have what we need
v = utils.get_vars(vtgate_port)
# some checks on performance / stats
# a typical workstation will do 38-40k QPS, check we have more than 10k
rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
if rpcCalls < 100000:
self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
utils.vtgate_kill(vtgate_proc)
def test_zkocc_qps(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(environment.zkocc_port_base)
if v['ZkReader']['test_nj']['State'] != 'Connected':
self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])
# some checks on performance / stats
# a typical workstation will do 45-47k QPS, check we have more than 10k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 100000:
self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
else:
logging.debug("Recorded qps: %u", rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 100000:
self.fail('Cache QPS is too low: %u < 10000' % (cacheReads / 10))
totalCacheReads = v['ZkReader']['total']['CacheReads']
self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
utils.zkocc_kill(zkocc_14850)
def test_regular_operation(self):
# Use a dedicated worker to run all vtworker commands.
worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(["--cell", "test_nj"], auto_log=True)
vtworker_endpoint = "localhost:" + str(worker_rpc_port)
automation_server_proc, automation_server_port = utils.run_automation_server()
source_shard_list = "0"
dest_shard_list = "-80,80-"
_, vtctld_endpoint = utils.vtctld.rpc_endpoint()
utils.run(
environment.binary_argstr("automation_client")
+ " --server localhost:"
+ str(automation_server_port)
+ " --task HorizontalReshardingTask"
+ " --param keyspace="
+ self.KEYSPACE
+ " --param source_shard_list="
+ source_shard_list
+ " --param dest_shard_list="
+ dest_shard_list
+ " --param vtctld_endpoint="
+ vtctld_endpoint
+ " --param vtworker_endpoint="
+ vtworker_endpoint
+ " --param min_healthy_rdonly_tablets=1"
)
self.verify()
utils.kill_sub_process(automation_server_proc, soft=True)
utils.kill_sub_process(worker_proc, soft=True)
def test_get_srv_keyspace(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s -vts-port %s %s' % (t.mysql_port, t.port + 500, t.tablet_alias))
self.rebuild()
# vtgate zk API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zk_port, trap_output=True)
self.assertEqual(err, "Partitions[master] =\n" +
" Shards[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" +
" Shards[0]={Start: , End: }\n" +
"Partitions[replica] =\n" +
" Shards[0]={Start: , End: }\n" +
"Shards[0]={Start: , End: }\n" +
"TabletTypes[0] = master\n",
"Got wrong content: %s" % err)
if environment.topo_server().flavor() == 'zookeeper':
reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace")
self.assertEqual(reply['TabletTypes'], ['master'])
# zkocc API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % environment.topo_server().zkocc_port_base, trap_output=True)
self.assertEqual(err,
"Partitions[master] =\n" +
" Shards[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" +
" Shards[0]={Start: , End: }\n" +
"Partitions[replica] =\n" +
" Shards[0]={Start: , End: }\n" +
"Shards[0]={Start: , End: }\n" +
"TabletTypes[0] = master\n",
"Got wrong content: %s" % err)
# vtgate zkocc API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zkocc_port, trap_output=True)
self.assertEqual(err, "Partitions[master] =\n" +
" Shards[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" +
" Shards[0]={Start: , End: }\n" +
"Partitions[replica] =\n" +
" Shards[0]={Start: , End: }\n" +
"Shards[0]={Start: , End: }\n" +
"TabletTypes[0] = master\n",
"Got wrong content: %s" % err)
def restore_customrules(self):
customrules = os.path.join(environment.tmproot, "customrules.json")
self.create_customrules(customrules)
if self.env == "vttablet":
if environment.topo_server().flavor() == "zookeeper":
utils.run(
environment.binary_argstr("zk") + " cp " + customrules + " /zk/test_ca/config/customrules/testrules"
)
def test_get_end_points(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
self.rebuild()
# vtgate zk API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zk_port, trap_output=True)
self.assertEqual(err, "Entries[0] = 1 localhost\n")
def zk_teardown():
global zk_port_base
zk_ports = ":".join(
[str(zk_port_base),
str(zk_port_base + 1),
str(zk_port_base + 2)])
run('%s -log_dir %s -zk.cfg 1@%s:%s teardown' %
(environment.binary_argstr('zkctl'), environment.vtlogroot, hostname,
zk_ports),
raise_on_error=False)
def test_get_end_points(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
self.rebuild()
# vtgate zk API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zk_port, trap_output=True)
self.assertEqual(err, "Entries[0] = 1 localhost\n")
def zk_teardown():
global zk_port_base
zk_ports = ":".join(
[str(zk_port_base),
str(zk_port_base + 1),
str(zk_port_base + 2)])
action = 'shutdown' if options.keep_logs else 'teardown'
run('%s -log_dir %s -zk.cfg 1@%s:%s %s' %
(environment.binary_argstr('zkctl'), environment.vtlogroot, hostname,
zk_ports, action),
raise_on_error=False)
def test_get_end_points(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
t.update_addrs()
self.rebuild()
# vtgate zk API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zk_port, trap_output=True)
self.assertEqual(err, "Entries[0] = 1 localhost\n")
if environment.topo_server().flavor() == 'zookeeper':
self.assertEqual(len(self.topo.get_end_points("test_nj", "test_keyspace", "0", "master")['Entries']), 1)
# zkocc API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % environment.topo_server().zkocc_port_base, trap_output=True)
self.assertEqual(err, "Entries[0] = 1 localhost\n")
# vtgate zkocc API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zkocc_port, trap_output=True)
self.assertEqual(err, "Entries[0] = 1 localhost\n")
def zk_setup(add_bad_host=False):
global zk_port_base
zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports))
config = environment.tmproot+'/test-zk-client-conf.json'
with open(config, 'w') as f:
ca_server = 'localhost:%u' % (zk_port_base+2)
if add_bad_host:
ca_server += ',does.not.exists:1234'
zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
'test_ny': 'localhost:%u'%(zk_port_base+2),
'test_ca': ca_server,
'global': 'localhost:%u'%(zk_port_base+2),
'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
'test_ca:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
'global:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),}
json.dump(zk_cell_mapping, f)
os.putenv('ZK_CLIENT_CONFIG', config)
run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt')
run(environment.binary_argstr('zk')+' touch -p /zk/test_ny/vt')
run(environment.binary_argstr('zk')+' touch -p /zk/test_ca/vt')
def zk_setup(add_bad_host=False):
global zk_port_base
zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports))
config = environment.tmproot+'/test-zk-client-conf.json'
with open(config, 'w') as f:
ca_server = 'localhost:%u' % (zk_port_base+2)
if add_bad_host:
ca_server += ',does.not.exists:1234'
zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
'test_ny': 'localhost:%u'%(zk_port_base+2),
'test_ca': ca_server,
'global': 'localhost:%u'%(zk_port_base+2),
'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
'test_ca:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
'global:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),}
json.dump(zk_cell_mapping, f)
os.environ['ZK_CLIENT_CONFIG'] = config
run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt')
run(environment.binary_argstr('zk')+' touch -p /zk/test_ny/vt')
run(environment.binary_argstr('zk')+' touch -p /zk/test_ca/vt')
def setUp(self):
environment.topo_server().wipe()
if environment.topo_server().flavor() == 'zookeeper':
utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc1')
utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc2')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename1 = fd.name
fd.write("Test data 1")
fd.close()
utils.run(environment.binary_argstr('zk')+' cp '+filename1+' /zk/test_nj/vt/zkocc1/data1')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename2 = fd.name
fd.write("Test data 2")
fd.close()
utils.run(environment.binary_argstr('zk')+' cp '+filename2+' /zk/test_nj/vt/zkocc1/data2')
fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename3 = fd.name
fd.write("Test data 3")
fd.close()
utils.run(environment.binary_argstr('zk')+' cp '+filename3+' /zk/test_nj/vt/zkocc1/data3')
def zk_wipe():
# Work around safety check on recursive delete.
run(environment.binary_argstr('zk')+' rm -rf /zk/test_nj/vt/*')
run(environment.binary_argstr('zk')+' rm -rf /zk/test_ny/vt/*')
run(environment.binary_argstr('zk')+' rm -rf /zk/global/vt/*')
run(environment.binary_argstr('zk')+' rm -f /zk/test_nj/vt')
run(environment.binary_argstr('zk')+' rm -f /zk/test_ny/vt')
run(environment.binary_argstr('zk')+' rm -f /zk/global/vt')
def zk_wipe():
# Work around safety check on recursive delete.
run(environment.binary_argstr('zk')+' rm -rf /zk/test_nj/vt/*')
run(environment.binary_argstr('zk')+' rm -rf /zk/test_ny/vt/*')
run(environment.binary_argstr('zk')+' rm -rf /zk/global/vt/*')
run(environment.binary_argstr('zk')+' rm -f /zk/test_nj/vt')
run(environment.binary_argstr('zk')+' rm -f /zk/test_ny/vt')
run(environment.binary_argstr('zk')+' rm -f /zk/global/vt')
def test_get_srv_keyspace_names(self):
utils.run_vtctl('CreateKeyspace test_keyspace1')
utils.run_vtctl('CreateKeyspace test_keyspace2')
t1 = tablet.Tablet(tablet_uid=1, cell="nj")
t1.init_tablet("master", "test_keyspace1", "0")
t1.update_addrs()
t2 = tablet.Tablet(tablet_uid=2, cell="nj")
t2.init_tablet("master", "test_keyspace2", "0")
t2.update_addrs()
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'], auto_log=True)
# vtgate API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getSrvKeyspaceNames test_nj' % utils.vtgate.port, trap_output=True)
self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
"KeyspaceNames[1] = test_keyspace2\n")
def change_customrules(self):
customrules = os.path.join(environment.tmproot, 'customrules.json')
with open(customrules, "w") as f:
f.write("""[{
"Name": "r2",
"Description": "disallow bindvar 'gfdsa'",
"BindVarConds":[{
"Name": "gfdsa",
"OnAbsent": false,
"Operator": "NOOP"
}]
}]""")
if self.env == "vttablet":
if environment.topo_server().flavor() == 'zookeeper':
utils.run(environment.binary_argstr('zk') + ' cp ' + customrules + ' /zk/test_ca/config/customrules/testrules')
def test_get_srv_keyspace(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s -vts-port %s %s' % (t.mysql_port, t.port + 500, t.tablet_alias))
self.rebuild()
# vtgate zk API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zk_port, trap_output=True)
self.assertEqual(err, "Partitions[master] =\n" +
" ShardReferences[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" +
" ShardReferences[0]={Start: , End: }\n" +
"Partitions[replica] =\n" +
" ShardReferences[0]={Start: , End: }\n",
"Got wrong content: %s" % err)
def test_get_srv_keyspace(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
t = tablet.Tablet(tablet_uid=1, cell="nj")
t.init_tablet("master", "test_keyspace", "0")
utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s %s' % (t.mysql_port, t.tablet_alias))
self.rebuild()
# vtgate zk API test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getSrvKeyspace test_nj test_keyspace' % utils.vtgate.port, trap_output=True)
self.assertEqual(err, "Partitions[master] =\n" +
" ShardReferences[0]={Start: , End: }\n" +
"Partitions[rdonly] =\n" +
" ShardReferences[0]={Start: , End: }\n" +
"Partitions[replica] =\n" +
" ShardReferences[0]={Start: , End: }\n",
"Got wrong content: %s" % err)
def test_get_srv_keyspace_names(self):
utils.run_vtctl('CreateKeyspace test_keyspace1')
utils.run_vtctl('CreateKeyspace test_keyspace2')
t1 = tablet.Tablet(tablet_uid=1, cell="nj")
t1.init_tablet("master", "test_keyspace1", "0")
t1.update_addrs()
t2 = tablet.Tablet(tablet_uid=2, cell="nj")
t2.init_tablet("master", "test_keyspace2", "0")
t2.update_addrs()
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'],
auto_log=True)
# vtgate API test
out, err = utils.run(
environment.binary_argstr('zkclient2') +
' -server localhost:%u -mode getSrvKeyspaceNames test_nj' %
utils.vtgate.port,
trap_output=True)
self.assertEqual(
err, "KeyspaceNames[0] = test_keyspace1\n" +
"KeyspaceNames[1] = test_keyspace2\n")
def zk_ls(path):
out, err = run(environment.binary_argstr('zk')+' ls '+path, trap_output=True)
return sorted(out.splitlines())
def test_sharding(self):
shard_0_master.init_tablet('master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet('master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets, wait for them to start
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.wait_for_vttablet_state('SERVING')
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias
])
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias
])
if environment.topo_server().flavor() == 'zookeeper':
# start zkocc, we'll use it later, indirectly with the vtdb-zkocc driver
zkocc_server = utils.zkocc_start()
# start vtgate, we'll use it later
vtgate_server, vtgate_port = utils.vtgate_start()
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.reset_replication()
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias
],
auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl([
'ApplySchemaShard', '-simple',
'-sql=' + create_vt_select_test_reverse.replace("\n", ""),
'test_keyspace/80-'
])
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl(['SetReadWrite', shard_0_master.tablet_alias])
utils.run_vtctl(['SetReadWrite', shard_1_master.tablet_alias])
shard_0_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (1, 'test 1')",
write=True)
shard_1_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (10, 'test 10')",
write=True)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
self._check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(
3803,
"/test_nj/test_keyspace/master",
"insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')",
bindvars='{"keyspace_id": 2}',
driver="vtdb",
verbose=True)
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery(
'vt_test_keyspace',
"select id, msg from vt_select_test order by id")
self.assertEqual(rows, (
(1, 'test 1'),
(2, 'test 2'),
), 'wrong mysql_query output: %s' % str(rows))
utils.pause("After db writes")
# now use various topo servers and streaming or both for the same query
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-streaming")
if environment.topo_server().flavor() == 'zookeeper':
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zk")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zk-streaming")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc")
self._check_rows(
["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc-streaming")
# make sure the schema checking works
self._check_rows_schema_diff("vtdb")
if environment.topo_server().flavor() == 'zookeeper':
self._check_rows_schema_diff("vtdb-zk")
self._check_rows_schema_diff("vtdb-zkocc")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/-80'])
utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/80-'])
out, err = utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
trap_output=True,
raise_on_error=False)
if 'test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err or \
'test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err:
self.fail('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl(['ValidateVersionShard', 'test_keyspace/-80'],
auto_log=True)
utils.run_vtctl(['ValidateVersionKeyspace', 'test_keyspace'],
auto_log=True)
# show and validate permissions
utils.run_vtctl(['GetPermissions', 'test_nj-0000062344'],
auto_log=True)
utils.run_vtctl(['ValidatePermissionsShard', 'test_keyspace/-80'],
auto_log=True)
utils.run_vtctl(['ValidatePermissionsKeyspace', 'test_keyspace'],
auto_log=True)
if environment.topo_server().flavor() == 'zookeeper':
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl(['ExportZknsForKeyspace', 'test_keyspace'])
out, err = utils.run(environment.binary_argstr('zk') +
' ls -R /zk/test_nj/zk?s/vt/test_keysp*',
trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']:
expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
if expected not in lines:
self.fail('missing zkns part:\n%s\nin:%s' %
(expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_keyspaces(vtgate_client)
shard_0_master_addrs = topology.get_host_port_by_name(
vtgate_client, "test_keyspace.-80.master:_vtocc")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# connect to shard -80
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"", "test_keyspace", "-80", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute(
"select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [
(1, 'test 1'),
(2, 'test 2'),
], 'wrong conn._execute output: %s' % str(results))
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(
vtgate_client, "test_keyspace.80-.master:_vtocc")
conn = tablet3.TabletConnection(
"%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]),
"", "test_keyspace", "80-", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute(
"select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [
(10, 'test 10'),
], 'wrong conn._execute output: %s' % str(results))
vtgate_client.close()
# try to connect with bad shard
try:
conn = tablet3.TabletConnection(
"localhost:%u" % shard_0_master.port, "", "test_keyspace",
"-90", 10.0)
conn.dial()
self.fail('expected an exception')
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(
e):
self.fail('unexpected exception: ' + str(e))
utils.vtgate_kill(vtgate_server)
if environment.topo_server().flavor() == 'zookeeper':
utils.kill_sub_process(zkocc_server)
tablet.kill_tablets(
[shard_0_master, shard_0_replica, shard_1_master, shard_1_replica])
def test_zkocc(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = environment.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, environment.topo_server().zkocc_port_base, bad_port+1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s' % str(e))
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
# FIXME(ryszard): This can be changed into a self.assertRaises.
try:
bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s' % str(e))
logging.getLogger().setLevel(level)
# get test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % environment.topo_server().zkocc_port_base, trap_output=True)
self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")
zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.assertDictContainsSubset({'Data': "Test data 1",
'Cached': True,
'Stale': False,},
zk_data)
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])
# getv test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % environment.topo_server().zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""")
zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
self.assertEqual(len(zk_data), 3)
for i, d in enumerate(zk_data):
self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
self.assertTrue(d['Cached'])
self.assertFalse(d['Stale'])
self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])
# children test
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode children /zk/test_nj/vt' % environment.topo_server().zkocc_port_base, trap_output=True)
self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 2
Cached = false
Stale = false
""")
# zk command tests
self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zk_data[0]['Stat']['MTime']),
_format_time(zk_data[1]['Stat']['MTime']),
_format_time(zk_data[2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/local/vt/zkocc1/data1' % environment.topo_server().zkocc_port_base, trap_output=True, raise_on_error=False)
self.assertIn("zkocc: cannot resolve local cell", err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
filename = outfd.name
querier = utils.run_bg('/bin/bash -c "while true ; do '+environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % environment.topo_server().zkocc_port_base, stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(environment.binary_argstr('zkctl')+' -zk.cfg 1@'+utils.hostname+':%s shutdown' % environment.topo_server().zk_ports)
time.sleep(3)
utils.run(environment.binary_argstr('zkctl')+' -zk.cfg 1@'+utils.hostname+':%s start' % environment.topo_server().zk_ports)
time.sleep(3)
utils.kill_sub_process(querier)
logging.debug("Checking %s", filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
self.fail('unexpected line: %s' % line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
self.fail('unexpected stale state')
self.assertEqual(state, 2)
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
level = logging.getLogger().getEffectiveLevel()
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if not str(e).startswith("Cannot dial to any server, tried: "):
self.fail('unexpected exception: %s', str(e))
logging.getLogger().setLevel(level)
def test_sharding(self):
shard_0_master.init_tablet('master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet('master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets, wait for them to start
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
# apply the schema on the first shard through vtctl, so all tablets
# are the same.
shard_0_master.mquery('vt_test_keyspace',
create_vt_select_test.replace('\n', ''), write=True)
shard_0_replica.mquery('vt_test_keyspace',
create_vt_select_test.replace('\n', ''), write=True)
# apply the schema on the second shard.
shard_1_master.mquery(
'vt_test_keyspace',
create_vt_select_test_reverse.replace('\n', ''), write=True)
shard_1_replica.mquery(
'vt_test_keyspace',
create_vt_select_test_reverse.replace('\n', ''), write=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
utils.run_vtctl(['ReloadSchema', t.tablet_alias])
# start vtgate, we'll use it later
utils.VtGate().start()
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl(['InitShardMaster', 'test_keyspace/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['InitShardMaster', 'test_keyspace/80-',
shard_1_master.tablet_alias], auto_log=True)
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl(['SetReadWrite', shard_0_master.tablet_alias])
utils.run_vtctl(['SetReadWrite', shard_1_master.tablet_alias])
shard_0_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (1, 'test 1')",
write=True)
shard_1_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (10, 'test 10')",
write=True)
utils.validate_topology(ping_tablets=True)
utils.pause('Before the sql scatter query')
# make sure the '1' value was written on first shard
rows = shard_0_master.mquery(
'vt_test_keyspace', 'select id, msg from vt_select_test order by id')
self.assertEqual(rows, ((1, 'test 1'),),
'wrong mysql_query output: %s' % str(rows))
utils.pause('After db writes')
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/-80'])
utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/80-'])
out, err = utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
trap_output=True, raise_on_error=False)
if ('test_nj-0000062344 and test_nj-0000062346 disagree on schema '
'for table vt_select_test:\nCREATE TABLE' not in err or
'test_nj-0000062344 and test_nj-0000062347 disagree on schema '
'for table vt_select_test:\nCREATE TABLE' not in err):
self.fail('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl(['ValidateVersionShard', 'test_keyspace/-80'],
auto_log=True)
utils.run_vtctl(['ValidateVersionKeyspace', 'test_keyspace'], auto_log=True)
# show and validate permissions
utils.run_vtctl(['GetPermissions', 'test_nj-0000062344'], auto_log=True)
utils.run_vtctl(['ValidatePermissionsShard', 'test_keyspace/-80'],
auto_log=True)
utils.run_vtctl(['ValidatePermissionsKeyspace', 'test_keyspace'],
auto_log=True)
if environment.topo_server().flavor() == 'zookeeper':
# and create zkns on this complex keyspace, make sure a few
# files are created
utils.run_vtctl(['ExportZknsForKeyspace', 'test_keyspace'])
out, err = utils.run(
environment.binary_argstr('zk') +
' ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/vt.vdns']:
expected = ('/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' +
db_type + sub_path)
if expected not in lines:
self.fail('missing zkns part:\n%s\nin:%s' %(expected, out))
# connect to the tablets directly, make sure they know / validate
# their own shard
sql = 'select id, msg from vt_select_test order by id'
qr = shard_0_master.execute(sql)
self.assertEqual(qr['Rows'], [['1', 'test 1'],])
qr = shard_1_master.execute(sql)
self.assertEqual(qr['Rows'], [['10', 'test 10'],])
_, stderr = utils.run_vtctl(['VtTabletExecute',
'-keyspace', 'test_keyspace',
'-shard', '-90',
shard_0_master.tablet_alias, sql],
expect_fail=True)
self.assertIn('fatal: Shard mismatch, expecting -80, received -90', stderr)
utils.vtgate.kill()
tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica])
def zk_teardown():
global zk_port_base
zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
run('%s -log_dir %s -zk.cfg 1@%s:%s teardown' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports), raise_on_error=False)
def zk_ls(path):
out, _ = run(environment.binary_argstr('zk') + ' ls ' + path,
trap_output=True)
return sorted(out.splitlines())
def _test_reparent_from_outside(self, brutal=False):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
tablet_41983.init_tablet('replica',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
tablet_31981.init_tablet('replica',
'test_keyspace',
'0',
start=True,
wait_for_start=False)
# wait for all tablets to start
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.wait_for_vttablet_state("SERVING")
# Reparent as a starting point
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' %
tablet_62344.tablet_alias,
auto_log=True)
# now manually reparent 1 out of 2 tablets
# 62044 will be the new master
# 31981 won't be re-parented, so it will be busted
tablet_62044.mquery('', mysql_flavor.promote_slave_commands())
new_pos = tablet_62044.mquery('', 'show master status')
logging.debug("New master position: %s" % str(new_pos))
# 62344 will now be a slave of 62044
tablet_62344.mquery('', [
"RESET MASTER", "RESET SLAVE",
"change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u"
% (utils.hostname, tablet_62044.mysql_port, new_pos[0][0],
new_pos[0][1]), 'start slave'
])
# 41983 will be a slave of 62044
tablet_41983.mquery('', [
'stop slave',
"change master to master_port=%u, master_log_file='%s', master_log_pos=%u"
% (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# in brutal mode, we scrap the old master first
if brutal:
tablet_62344.scrap(force=True)
# we have some automated tools that do this too, so it's good to simulate
if environment.topo_server_implementation == 'zookeeper':
utils.run(
environment.binary_argstr('zk') + ' rm -rf ' +
tablet_62344.zk_tablet_path)
# update zk with the new graph
utils.run_vtctl('ShardExternallyReparented test_keyspace/0 %s' %
tablet_62044.tablet_alias,
auto_log=True)
self._test_reparent_from_outside_check(brutal)
utils.run_vtctl('RebuildReplicationGraph test_nj test_keyspace')
self._test_reparent_from_outside_check(brutal)
tablet.kill_tablets(
[tablet_31981, tablet_62344, tablet_62044, tablet_41983])
def zk_cat(path):
out, _ = run(environment.binary_argstr("zk") + " cat " + path, trap_output=True)
return out
def test_sharding(self):
shard_0_master.init_tablet( 'master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet( 'master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets, wait for them to start
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state('SERVING')
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl(['ApplySchema',
'-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias])
utils.run_vtctl(['ApplySchema',
'-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias])
# start vtgate, we'll use it later
vtgate_server, vtgate_port = utils.vtgate_start()
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl(['InitShardMaster', 'test_keyspace/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['InitShardMaster', 'test_keyspace/80-',
shard_1_master.tablet_alias], auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl(['ApplySchemaShard',
'-simple',
'-sql=' + create_vt_select_test_reverse.replace("\n", ""),
'test_keyspace/80-'])
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl(['SetReadWrite', shard_0_master.tablet_alias])
utils.run_vtctl(['SetReadWrite', shard_1_master.tablet_alias])
shard_0_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True)
shard_1_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(3803, "/test_nj/test_keyspace/master", "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')", bindvars='{"keyspace_id": 2}', driver="vtdb", verbose=True)
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery('vt_test_keyspace', "select id, msg from vt_select_test order by id")
self.assertEqual(rows, ((1, 'test 1'), (2, 'test 2'), ),
'wrong mysql_query output: %s' % str(rows))
utils.pause("After db writes")
# now use various topo servers and streaming or both for the same query
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-streaming")
if environment.topo_server().flavor() == 'zookeeper':
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-zk")
self._check_rows(["Index\tid\tmsg",
"1\ttest 1",
"2\ttest 2",
"10\ttest 10"],
driver="vtdb-zk-streaming")
# make sure the schema checking works
self._check_rows_schema_diff("vtdb")
if environment.topo_server().flavor() == 'zookeeper':
self._check_rows_schema_diff("vtdb-zk")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/-80'])
utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/80-'])
out, err = utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
trap_output=True, raise_on_error=False)
if 'test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err or \
'test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err:
self.fail('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl(['ValidateVersionShard', 'test_keyspace/-80'],
auto_log=True)
utils.run_vtctl(['ValidateVersionKeyspace', 'test_keyspace'], auto_log=True)
# show and validate permissions
utils.run_vtctl(['GetPermissions', 'test_nj-0000062344'], auto_log=True)
utils.run_vtctl(['ValidatePermissionsShard', 'test_keyspace/-80'],
auto_log=True)
utils.run_vtctl(['ValidatePermissionsKeyspace', 'test_keyspace'],
auto_log=True)
if environment.topo_server().flavor() == 'zookeeper':
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl(['ExportZknsForKeyspace', 'test_keyspace'])
out, err = utils.run(environment.binary_argstr('zk')+' ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/vt.vdns']:
expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
if expected not in lines:
self.fail('missing zkns part:\n%s\nin:%s' %(expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
topology.read_keyspaces(vtgate_client)
shard_0_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.-80.master:vt")
if len(shard_0_master_addrs) != 1:
self.fail('topology.get_host_port_by_name failed for "test_keyspace.-80.master:vt", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# connect to shard -80
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0],
shard_0_master_addrs[0][1]),
"", "test_keyspace", "-80", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [(1, 'test 1'), (2, 'test 2'), ],
'wrong conn._execute output: %s' % str(results))
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.80-.master:vt")
conn = tablet3.TabletConnection("%s:%u" % (shard_1_master_addrs[0][0],
shard_1_master_addrs[0][1]),
"", "test_keyspace", "80-", 10.0)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
self.assertEqual(results, [(10, 'test 10'), ],
'wrong conn._execute output: %s' % str(results))
vtgate_client.close()
# try to connect with bad shard
try:
conn = tablet3.TabletConnection("localhost:%u" % shard_0_master.port,
"", "test_keyspace", "-90", 10.0)
conn.dial()
self.fail('expected an exception')
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(e):
self.fail('unexpected exception: ' + str(e))
utils.vtgate_kill(vtgate_server)
tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica])
def test_sharding(self):
shard_0_master.init_tablet("master", "test_keyspace", "-80")
shard_0_replica.init_tablet("replica", "test_keyspace", "-80")
shard_1_master.init_tablet("master", "test_keyspace", "80-")
shard_1_replica.init_tablet("replica", "test_keyspace", "80-")
utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets, wait for them to start
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.create_db("vt_test_keyspace")
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.wait_for_vttablet_state("SERVING")
# apply the schema on the first shard through vtctl, so all tablets
# are the same.
shard_0_master.mquery("vt_test_keyspace", create_vt_select_test.replace("\n", ""), write=True)
shard_0_replica.mquery("vt_test_keyspace", create_vt_select_test.replace("\n", ""), write=True)
# apply the schema on the second shard.
shard_1_master.mquery("vt_test_keyspace", create_vt_select_test_reverse.replace("\n", ""), write=True)
shard_1_replica.mquery("vt_test_keyspace", create_vt_select_test_reverse.replace("\n", ""), write=True)
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
utils.run_vtctl(["ReloadSchema", t.tablet_alias])
# start vtgate, we'll use it later
utils.VtGate().start()
for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
t.reset_replication()
utils.run_vtctl(["InitShardMaster", "test_keyspace/-80", shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(["InitShardMaster", "test_keyspace/80-", shard_1_master.tablet_alias], auto_log=True)
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl(["SetReadWrite", shard_0_master.tablet_alias])
utils.run_vtctl(["SetReadWrite", shard_1_master.tablet_alias])
shard_0_master.mquery(
"vt_test_keyspace", "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True
)
shard_1_master.mquery(
"vt_test_keyspace", "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True
)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# make sure the '1' value was written on first shard
rows = shard_0_master.mquery("vt_test_keyspace", "select id, msg from vt_select_test order by id")
self.assertEqual(rows, ((1, "test 1"),), "wrong mysql_query output: %s" % str(rows))
utils.pause("After db writes")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl(["ValidateSchemaShard", "test_keyspace/-80"])
utils.run_vtctl(["ValidateSchemaShard", "test_keyspace/80-"])
out, err = utils.run_vtctl(["ValidateSchemaKeyspace", "test_keyspace"], trap_output=True, raise_on_error=False)
if (
"test_nj-0000062344 and test_nj-0000062346 disagree on schema "
"for table vt_select_test:\nCREATE TABLE" not in err
or "test_nj-0000062344 and test_nj-0000062347 disagree on schema "
"for table vt_select_test:\nCREATE TABLE" not in err
):
self.fail("wrong ValidateSchemaKeyspace output: " + err)
# validate versions
utils.run_vtctl(["ValidateVersionShard", "test_keyspace/-80"], auto_log=True)
utils.run_vtctl(["ValidateVersionKeyspace", "test_keyspace"], auto_log=True)
# show and validate permissions
utils.run_vtctl(["GetPermissions", "test_nj-0000062344"], auto_log=True)
utils.run_vtctl(["ValidatePermissionsShard", "test_keyspace/-80"], auto_log=True)
utils.run_vtctl(["ValidatePermissionsKeyspace", "test_keyspace"], auto_log=True)
if environment.topo_server().flavor() == "zookeeper":
# and create zkns on this complex keyspace, make sure a few
# files are created
utils.run_vtctl(["ExportZknsForKeyspace", "test_keyspace"])
out, err = utils.run(
environment.binary_argstr("zk") + " ls -R /zk/test_nj/zk?s/vt/test_keysp*", trap_output=True
)
lines = out.splitlines()
for base in ["-80", "80-"]:
for db_type in ["master", "replica"]:
for sub_path in ["", ".vdns", "/0", "/vt.vdns"]:
expected = "/zk/test_nj/zkns/vt/test_keyspace/" + base + "/" + db_type + sub_path
if expected not in lines:
self.fail("missing zkns part:\n%s\nin:%s" % (expected, out))
# connect to the tablets directly, make sure they know / validate
# their own shard
sql = "select id, msg from vt_select_test order by id"
qr = shard_0_master.execute(sql)
self.assertEqual(qr["Rows"], [["1", "test 1"]])
qr = shard_1_master.execute(sql)
self.assertEqual(qr["Rows"], [["10", "test 10"]])
_, stderr = utils.run_vtctl(
["VtTabletExecute", "-keyspace", "test_keyspace", "-shard", "-90", shard_0_master.tablet_alias, sql],
expect_fail=True,
)
self.assertIn("fatal: Shard mismatch, expecting -80, received -90", stderr)
utils.vtgate.kill()
tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master, shard_1_replica])
def zk_cat(path):
out, err = run(environment.binary_argstr('zk')+' cat '+path, trap_output=True)
return out
def start_vttablet(self,
port=None,
auth=False,
memcache=False,
wait_for_state="SERVING",
customrules=None,
schema_override=None,
cert=None,
key=None,
ca_cert=None,
repl_extra_flags={},
table_acl_config=None,
target_tablet_type=None,
lameduck_period=None,
extra_args=None,
full_mycnf_args=False,
security_policy=None):
"""
Starts a vttablet process, and returns it.
The process is also saved in self.proc, so it's easy to kill as well.
"""
environment.prog_compile('vtaction')
args = environment.binary_args('vttablet') + [
'-port',
'%s' % (port or self.port), '-tablet-path', self.tablet_alias,
'-log_dir', environment.vtlogroot
]
args.extend(environment.topo_server_flags())
args.extend(utils.binlog_player_protocol_flags)
dbconfigs = self._get_db_configs_file(repl_extra_flags)
for key1 in dbconfigs:
for key2 in dbconfigs[key1]:
args.extend(
["-db-config-" + key1 + "-" + key2, dbconfigs[key1][key2]])
if full_mycnf_args:
# this flag is used to specify all the mycnf_ flags, to make
# sure that code works and can fork actions.
relay_log_path = os.path.join(
self.tablet_dir, "relay-logs",
"vt-%010d-relay-bin" % self.tablet_uid)
args.extend([
"-mycnf_server_id",
str(self.tablet_uid),
"-mycnf_mysql_port",
str(self.mysql_port),
"-mycnf_data_dir",
os.path.join(self.tablet_dir, "data"),
"-mycnf_innodb_data_home_dir",
os.path.join(self.tablet_dir, "innodb", "data"),
"-mycnf_innodb_log_group_home_dir",
os.path.join(self.tablet_dir, "innodb", "logs"),
"-mycnf_socket_file",
os.path.join(self.tablet_dir, "mysql.sock"),
"-mycnf_error_log_path",
os.path.join(self.tablet_dir, "error.log"),
"-mycnf_slow_log_path",
os.path.join(self.tablet_dir, "slow-query.log"),
"-mycnf_relay_log_path",
relay_log_path,
"-mycnf_relay_log_index_path",
relay_log_path + ".index",
"-mycnf_relay_log_info_path",
os.path.join(self.tablet_dir, "relay-logs", "relay-log.info"),
"-mycnf_bin_log_path",
os.path.join(self.tablet_dir, "bin-logs",
"vt-%010d-bin" % self.tablet_uid),
"-mycnf_master_info_file",
os.path.join(self.tablet_dir, "master.info"),
"-mycnf_pid_file",
os.path.join(self.tablet_dir, "mysql.pid"),
"-mycnf_tmp_dir",
os.path.join(self.tablet_dir, "tmp"),
"-mycnf_slave_load_tmp_dir",
os.path.join(self.tablet_dir, "tmp"),
])
if memcache:
args.extend(["-rowcache-bin", environment.memcached_bin()])
memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
args.extend(["-rowcache-socket", memcache_socket])
args.extend(["-enable-rowcache"])
if auth:
args.extend([
'-auth-credentials',
os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json')
])
if customrules:
args.extend(['-customrules', customrules])
if schema_override:
args.extend(['-schema-override', schema_override])
if table_acl_config:
args.extend(['-table-acl-config', table_acl_config])
args.extend(['-queryserver-config-strict-table-acl'])
if cert:
self.secure_port = environment.reserve_ports(1)
args.extend([
'-secure-port',
'%s' % self.secure_port, '-cert', cert, '-key', key
])
if ca_cert:
args.extend(['-ca_cert', ca_cert])
if target_tablet_type:
args.extend([
'-target_tablet_type', target_tablet_type,
'-health_check_interval', '2s', '-allowed_replication_lag',
'30'
])
if lameduck_period:
args.extend(['-lameduck-period', lameduck_period])
if extra_args:
args.extend(extra_args)
if security_policy:
args.extend(['-security_policy', security_policy])
stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
# increment count only the first time
if not self.proc:
Tablet.tablets_running += 1
self.proc = utils.run_bg(args, stderr=stderr_fd)
stderr_fd.close()
# wait for zookeeper PID just to be sure we have it
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_argstr('zk') + ' wait -e ' +
self.zk_pid,
stdout=utils.devnull)
# wait for query service to be in the right state
if wait_for_state:
self.wait_for_vttablet_state(wait_for_state, port=port)
return self.proc
def zk_teardown():
global zk_port_base
zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
action = 'shutdown' if options.keep_logs else 'teardown'
run('%s -log_dir %s -zk.cfg 1@%s:%s %s' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports, action), raise_on_error=False)
def _test_reparent_from_outside(self, brutal=False):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
# wait for all tablets to start
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.wait_for_vttablet_state("SERVING")
# Reparent as a starting point
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias, auto_log=True)
# now manually reparent 1 out of 2 tablets
# 62044 will be the new master
# 31981 won't be re-parented, so it will be busted
tablet_62044.mquery('', [
"RESET MASTER",
"STOP SLAVE",
"RESET SLAVE",
"CHANGE MASTER TO MASTER_HOST = ''",
])
new_pos = tablet_62044.mquery('', 'show master status')
logging.debug("New master position: %s" % str(new_pos))
# 62344 will now be a slave of 62044
tablet_62344.mquery('', [
"RESET MASTER",
"RESET SLAVE",
"change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# 41983 will be a slave of 62044
tablet_41983.mquery('', [
'stop slave',
"change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# in brutal mode, we scrap the old master first
if brutal:
tablet_62344.scrap(force=True)
# we have some automated tools that do this too, so it's good to simulate
if environment.topo_server_implementation == 'zookeeper':
utils.run(environment.binary_argstr('zk')+' rm -rf ' + tablet_62344.zk_tablet_path)
# update zk with the new graph
utils.run_vtctl('ShardExternallyReparented test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True)
self._test_reparent_from_outside_check(brutal)
utils.run_vtctl('RebuildReplicationGraph test_nj test_keyspace')
self._test_reparent_from_outside_check(brutal)
tablet.kill_tablets([tablet_31981, tablet_62344, tablet_62044, tablet_41983])
def zk_cat(path):
out, _ = run(environment.binary_argstr('zk') + ' cat ' + path,
trap_output=True)
return out
def zk_ls(path):
out, _ = run(environment.binary_argstr("zk") + " ls " + path, trap_output=True)
return sorted(out.splitlines())
