def setUpModule():
try:
environment.topo_server().setup()
setup_topology()
# start mysql instance external to the test
global __tablets
setup_procs = []
for tablet in __tablets:
setup_procs.append(tablet.init_mysql())
utils.wait_procs(setup_procs)
create_db()
start_tablets()
utils.VtGate().start()
except:
tearDownModule()
raise
def main():
parser = optparse.OptionParser(usage='usage: %prog [options]')
utils.add_options(parser)
(options, args) = parser.parse_args()
options.debug = True
utils.set_options(options)
env = keyspace_util.TestEnv()
try:
environment.topo_server().setup()
env.launch(
'user',
shards=['-80', '80-'],
ddls=[
'create table user(user_id bigint, name varchar(128), '
'primary key(user_id))',
'create table user_extra(user_id bigint, extra varchar(128), '
'primary key(user_id))',
'create table music(user_id bigint, music_id bigint, '
'primary key(user_id, music_id))',
'create table music_extra(music_id bigint, '
'keyspace_id bigint unsigned, primary key(music_id))',
],
)
env.launch(
'lookup',
ddls=[
'create table user_idx(user_id bigint not null auto_increment, '
'primary key(user_id))',
'create table name_user_idx(name varchar(128), user_id bigint, '
'primary key(name, user_id))',
'create table music_user_idx(music_id bigint not null '
'auto_increment, user_id bigint, primary key(music_id))',
],
)
utils.apply_vschema(vschema)
utils.VtGate().start(cache_ttl='500s')
utils.Vtctld().start()
print 'vtgate:', utils.vtgate.port
print 'vtctld:', utils.vtctld.port
utils.pause('the cluster is up, press enter to shut it down...')
finally:
env.teardown()
utils.kill_sub_processes()
utils.remove_tmp_files()
environment.topo_server().teardown()
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [master_tablet.init_mysql(),
replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# start a vtctld so the vtctl insert commands are just RPCs, not forks.
utils.Vtctld().start()
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0', tablet_index=0)
replica_tablet.init_tablet('replica', 'test_keyspace', '0', tablet_index=1)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('vt_test_keyspace')
master_tablet.start_vttablet(wait_for_state=None)
replica_tablet.start_vttablet(wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('NOT_SERVING')
utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
master_tablet.tablet_alias], auto_log=True)
utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica')
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
master_tablet.mquery('vt_test_keyspace', _create_vt_a)
master_tablet.mquery('vt_test_keyspace', _create_vt_b)
utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
utils.run_vtctl(['RebuildVSchemaGraph'])
utils.VtGate().start(tablets=[master_tablet, replica_tablet])
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
create_join_user,
create_join_user_extra,
create_join_name_info,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_vt_user_seq,
create_vt_music_seq,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start()
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
source_master.init_mysql(),
source_replica.init_mysql(),
source_rdonly1.init_mysql(),
source_rdonly2.init_mysql(),
destination_master.init_mysql(),
destination_replica.init_mysql(),
destination_rdonly1.init_mysql(),
destination_rdonly2.init_mysql(),
]
utils.Vtctld().start()
utils.VtGate().start(cache_ttl='0s')
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_0_replica
global shard_1_master
global lookup_master
logging.debug('in setUpModule')
try:
environment.topo_server().setup()
logging.debug('Setting up tablets')
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
'user',
shards=['-80', '80-'],
ddls=[
create_sharded_message,
],
)
keyspace_env.launch(
'lookup',
ddls=[
create_unsharded_message,
],
)
shard_0_master = keyspace_env.tablet_map['user.-80.master']
shard_0_replica = keyspace_env.tablet_map['user.-80.replica.0']
shard_1_master = keyspace_env.tablet_map['user.80-.master']
lookup_master = keyspace_env.tablet_map['lookup.0.master']
utils.apply_vschema(vschema)
utils.VtGate().start(tablets=[
shard_0_master, shard_0_replica, shard_1_master, lookup_master
])
utils.vtgate.wait_for_endpoints('user.-80.master', 1)
utils.vtgate.wait_for_endpoints('user.-80.replica', 1)
utils.vtgate.wait_for_endpoints('user.80-.master', 1)
utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
try:
environment.topo_server().setup()
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch('user', ddls=[
create_vt_user,
create_main,
])
shard_0_master = keyspace_env.tablet_map['user.0.master']
utils.VtGate().start(
tablets=[shard_0_master],
extra_args=['-vschema_ddl_authorized_users', '%'],
)
utils.vtgate.wait_for_endpoints('user.0.master', 1)
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# start a vtctld so the vtctl insert commands are just RPCs, not forks
utils.Vtctld().start()
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
utils.validate_topology()
master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)
utils.VtGate().start()
master_tablet.start_vttablet(memcache=True, wait_for_state=None)
replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
utils.run_vtctl(
['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias],
auto_log=True)
utils.validate_topology()
# restart the replica tablet so the stats are reset
replica_tablet.kill_vttablet()
replica_tablet.start_vttablet(memcache=True)
except:
tearDownModule()
raise
def setUpModule():
try:
environment.topo_server().setup()
setup_procs = [
shard_master.init_mysql(),
shard_replica.init_mysql(),
shard_rdonly1.init_mysql(),
shard_0_master.init_mysql(),
shard_0_replica.init_mysql(),
shard_0_rdonly1.init_mysql(),
shard_1_master.init_mysql(),
shard_1_replica.init_mysql(),
shard_1_rdonly1.init_mysql(),
]
# we only use vtgate for testing SplitQuery works correctly.
# we want cache_ttl at zero so we re-read the topology for every test query.
utils.VtGate().start(cache_ttl='0')
utils.wait_procs(setup_procs)
except:
tearDownModule()
raise
def setUpModule():
global keyspace_env
global shard_0_master
global shard_1_master
global lookup_master
logging.debug("in setUpModule")
try:
environment.topo_server().setup()
logging.debug("Setting up tablets")
keyspace_env = keyspace_util.TestEnv()
keyspace_env.launch(
"user",
shards=["-80", "80-"],
ddls=[
create_vt_user,
create_vt_user2,
create_vt_user_extra,
create_vt_music,
create_vt_music_extra,
],
)
keyspace_env.launch(
"lookup",
ddls=[
create_vt_user_idx,
create_music_user_map,
create_name_user2_map,
],
)
shard_0_master = keyspace_env.tablet_map["user.-80.master"]
shard_1_master = keyspace_env.tablet_map["user.80-.master"]
lookup_master = keyspace_env.tablet_map["lookup.0.master"]
utils.apply_vschema(schema)
utils.VtGate().start()
except:
tearDownModule()
raise
def setup_tablets():
setup_sharded_keyspace()
setup_unsharded_keyspace()
utils.VtGate().start(tablets=[
shard_0_master,
shard_0_replica,
shard_1_master,
shard_1_replica,
unsharded_master,
unsharded_replica,
])
utils.vtgate.wait_for_endpoints('%s.%s.master' % (SHARDED_KEYSPACE, '80-'),
1)
utils.vtgate.wait_for_endpoints(
'%s.%s.replica' % (SHARDED_KEYSPACE, '80-'), 1)
utils.vtgate.wait_for_endpoints('%s.%s.master' % (SHARDED_KEYSPACE, '-80'),
1)
utils.vtgate.wait_for_endpoints(
'%s.%s.replica' % (SHARDED_KEYSPACE, '-80'), 1)
utils.vtgate.wait_for_endpoints('%s.%s.master' % (UNSHARDED_KEYSPACE, '0'),
1)
utils.vtgate.wait_for_endpoints(
'%s.%s.replica' % (UNSHARDED_KEYSPACE, '0'), 1)
def test_multi_recovery(self):
"""Test recovery from backup flow.
test_multi_recovery will:
- create a shard with master and replica1 only
- run InitShardMaster
- insert some data
- take a backup
- insert more data on the master
- take another backup
- create a recovery keyspace after first backup
- bring up tablet_replica2 in the new keyspace
- check that new tablet does not have data created after backup1
- create second recovery keyspace after second backup
- bring up tablet_replica3 in second keyspace
- check that new tablet has data created after backup1 but not data created after backup2
- check that vtgate queries work correctly
"""
# insert data on master, wait for replica to get it
utils.run_vtctl([
'ApplySchema', '-sql', self._create_vt_insert_test, 'test_keyspace'
],
auto_log=True)
self._insert_data(tablet_master, 1)
self._check_data(tablet_replica1, 1, 'replica1 tablet getting data')
# backup the replica
utils.run_vtctl(['Backup', tablet_replica1.tablet_alias],
auto_log=True)
# check that the backup shows up in the listing
backups = self._list_backups()
logging.debug('list of backups: %s', backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].endswith(tablet_replica1.tablet_alias))
# insert more data on the master
self._insert_data(tablet_master, 2)
# wait for it to replicate
self._check_data(tablet_replica1, 2, 'replica1 tablet getting data')
utils.run_vtctl(
['ApplyVSchema', '-vschema', self._vschema_json, 'test_keyspace'],
auto_log=True)
vs = utils.run_vtctl_json(['GetVSchema', 'test_keyspace'])
logging.debug('test_keyspace vschema: %s', str(vs))
# now bring up the other replica, letting it restore from backup.
self._restore(tablet_replica2, 'recovery_ks1')
# we are not asserting on the contents of vschema here
# because the later part of the test (vtgate) will fail
# if the vschema is not copied correctly from the base_keyspace
vs = utils.run_vtctl_json(['GetVSchema', 'recovery_ks1'])
logging.debug('recovery_ks1 vschema: %s', str(vs))
# check the new replica does not have the data
self._check_data(tablet_replica2, 1,
'replica2 tablet should not have new data')
# update original 1st row in master
tablet_master.mquery(
'vt_test_keyspace',
"update vt_insert_test set msg='new msg 1' where id=1",
write=True)
# verify that master has new value
result = tablet_master.mquery(
'vt_test_keyspace', 'select msg from vt_insert_test where id=1')
self.assertEqual(result[0][0], 'new msg 1')
# verify that restored replica has old value
result = tablet_replica2.mquery(
'vt_test_keyspace', 'select msg from vt_insert_test where id=1')
self.assertEqual(result[0][0], 'test 1')
# take another backup on the replica
utils.run_vtctl(['Backup', tablet_replica1.tablet_alias],
auto_log=True)
# insert more data on the master
self._insert_data(tablet_master, 3)
# wait for it to replicate
self._check_data(tablet_replica1, 3, 'replica1 tablet getting data')
# now bring up the other replica, letting it restore from backup2.
# this also validates that if there are multiple backups, the most recent one is used
self._restore(tablet_replica3, 'recovery_ks2')
vs = utils.run_vtctl(['GetVSchema', 'recovery_ks2'])
logging.debug('recovery_ks2 vschema: %s', str(vs))
# check the new replica does not have the latest data
self._check_data(tablet_replica3, 2,
'replica3 tablet should not have new data')
# update original 1st row in master again
tablet_master.mquery(
'vt_test_keyspace',
"update vt_insert_test set msg='new msg 2' where id=1",
write=True)
# verify that master has new value
result = tablet_master.mquery(
'vt_test_keyspace', 'select msg from vt_insert_test where id=1')
self.assertEqual(result[0][0], 'new msg 2')
# verify that restored replica has correct value
result = tablet_replica3.mquery(
'vt_test_keyspace', 'select msg from vt_insert_test where id=1')
self.assertEqual(result[0][0], 'new msg 1')
# start vtgate
vtgate = utils.VtGate()
vtgate.start(tablets=all_tablets, tablet_types_to_wait='REPLICA')
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
utils.vtgate.wait_for_endpoints('recovery_ks1.0.replica', 1)
utils.vtgate.wait_for_endpoints('recovery_ks2.0.replica', 1)
# check that vtgate doesn't route queries to new tablet
vtgate_conn = get_connection()
cursor = vtgate_conn.cursor(tablet_type='replica',
keyspace=None,
writable=True)
cursor.execute('select count(*) from vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 3)
cursor.execute('select msg from vt_insert_test where id=1', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 'new msg 2')
# check that new keyspace is accessible by using ks.table
cursor.execute('select count(*) from recovery_ks1.vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 1)
cursor.execute(
'select msg from recovery_ks1.vt_insert_test where id=1', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 'test 1')
# check that new keyspace is accessible by using ks.table
cursor.execute('select count(*) from recovery_ks2.vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 2)
cursor.execute(
'select msg from recovery_ks2.vt_insert_test where id=1', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 'new msg 1')
# TODO check that new tablet is accessible with 'use ks:shard'
# this currently does not work through the python client, though it works from mysql client
#cursor.execute('use recovery_ks1:0@replica', {})
#cursor.execute('select count(*) from vt_insert_test', {})
#result = cursor.fetchall()
#if not result:
#self.fail('Result cannot be null')
#else:
#self.assertEqual(result[0][0], 1)
vtgate_conn.close()
vtgate.kill()
def test_basic_recovery(self):
"""Test recovery from backup flow.
test_recovery will:
- create a shard with master and replica1 only
- run InitShardMaster
- insert some data
- take a backup
- insert more data on the master
- create a recovery keyspace
- bring up tablet_replica2 in the new keyspace
- check that new tablet does not have data created after backup
- check that vtgate queries work correctly
"""
# insert data on master, wait for replica to get it
utils.run_vtctl([
'ApplySchema', '-sql', self._create_vt_insert_test, 'test_keyspace'
],
auto_log=True)
self._insert_data(tablet_master, 1)
self._check_data(tablet_replica1, 1, 'replica1 tablet getting data')
master_pos = mysql_flavor().master_position(tablet_master)
# backup the replica
utils.run_vtctl(['Backup', tablet_replica1.tablet_alias],
auto_log=True)
# check that the backup shows up in the listing
backups = self._list_backups()
logging.debug('list of backups: %s', backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].endswith(tablet_replica1.tablet_alias))
# backup name is of format date.time.tablet_alias
strs = backups[0].split('.')
expectedTime = datetime.strptime(strs[0] + '.' + strs[1],
'%Y-%m-%d.%H%M%S')
# insert more data on the master
self._insert_data(tablet_master, 2)
utils.run_vtctl(
['ApplyVSchema', '-vschema', self._vschema_json, 'test_keyspace'],
auto_log=True)
vs = utils.run_vtctl_json(['GetVSchema', 'test_keyspace'])
logging.debug('test_keyspace vschema: %s', str(vs))
ks = utils.run_vtctl_json(
['GetSrvKeyspace', 'test_nj', 'test_keyspace'])
logging.debug('Serving keyspace before: %s', str(ks))
vs = utils.run_vtctl_json(['GetSrvVSchema', 'test_nj'])
logging.debug('Serving vschema before recovery: %s', str(vs))
# now bring up the recovery keyspace with 1 tablet, letting it restore from backup.
self._restore(tablet_replica2, 'recovery_keyspace')
vs = utils.run_vtctl_json(['GetSrvVSchema', 'test_nj'])
logging.debug('Serving vschema after recovery: %s', str(vs))
ks = utils.run_vtctl_json(
['GetSrvKeyspace', 'test_nj', 'test_keyspace'])
logging.debug('Serving keyspace after: %s', str(ks))
vs = utils.run_vtctl_json(['GetVSchema', 'recovery_keyspace'])
logging.debug('recovery_keyspace vschema: %s', str(vs))
# check the new replica has only 1 row
self._check_data(tablet_replica2, 1,
'replica2 tablet should not have new data')
# check that the restored replica has the right local_metadata
result = tablet_replica2.mquery('_vt', 'select * from local_metadata')
metadata = {}
for row in result:
metadata[row[0]] = row[1]
self.assertEqual(metadata['Alias'], 'test_nj-0000062346')
self.assertEqual(metadata['ClusterAlias'], 'recovery_keyspace.0')
self.assertEqual(metadata['DataCenter'], 'test_nj')
self.assertEqual(metadata['RestorePosition'], master_pos)
logging.debug('RestoredBackupTime: %s',
str(metadata['RestoredBackupTime']))
gotTime = datetime.strptime(metadata['RestoredBackupTime'],
'%Y-%m-%dT%H:%M:%SZ')
self.assertEqual(gotTime, expectedTime)
# update original 1st row in master
tablet_master.mquery(
'vt_test_keyspace',
"update vt_insert_test set msg='new msg' where id=1",
write=True)
# verify that master has new value
result = tablet_master.mquery(
'vt_test_keyspace', 'select msg from vt_insert_test where id=1')
self.assertEqual(result[0][0], 'new msg')
# verify that restored replica has old value
result = tablet_replica2.mquery(
'vt_test_keyspace', 'select msg from vt_insert_test where id=1')
self.assertEqual(result[0][0], 'test 1')
# start vtgate
vtgate = utils.VtGate()
vtgate.start(tablets=[tablet_master, tablet_replica1, tablet_replica2],
tablet_types_to_wait='REPLICA')
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
utils.vtgate.wait_for_endpoints('recovery_keyspace.0.replica', 1)
# check that vtgate doesn't route queries to new tablet
vtgate_conn = get_connection()
cursor = vtgate_conn.cursor(tablet_type='replica',
keyspace=None,
writable=True)
cursor.execute('select count(*) from vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 2)
cursor.execute('select msg from vt_insert_test where id=1', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 'new msg')
# check that new keyspace is accessible by using ks.table
cursor.execute('select count(*) from recovery_keyspace.vt_insert_test',
{})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 1)
cursor.execute(
'select msg from recovery_keyspace.vt_insert_test where id=1', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 'test 1')
# check that new keyspace is accessible with 'use ks'
cursor.execute('use recovery_keyspace@replica', {})
cursor.execute('select count(*) from vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 1)
cursor.execute(
'select msg from recovery_keyspace.vt_insert_test where id=1', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 'test 1')
# TODO check that new tablet is accessible with 'use ks:shard'
# this currently does not work through the python client, though it works from mysql client
#cursor.execute('use recovery_keyspace:0@replica', {})
#cursor.execute('select count(*) from vt_insert_test', {})
#result = cursor.fetchall()
#if not result:
#self.fail('Result cannot be null')
#else:
#self.assertEqual(result[0][0], 1)
vtgate_conn.close()
tablet_replica2.kill_vttablet()
vtgate.kill()
def test_secure(self):
utils.VtGate().start(cache_ttl='0s')
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert':
cert_dir + "/client-cert.pem",
'ssl-key':
cert_dir + "/client-key.pem",
})
# Reparent using SSL (this will also check replication works)
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl([
'InitShardMaster', 'test_keyspace/0', shard_0_master.tablet_alias
],
auto_log=True)
# then get the topology and check it
topo_client = zkocc.ZkOccConnection(utils.vtgate.addr(), "test_nj",
30.0)
topology.read_keyspaces(topo_client)
shard_0_master_addrs = topology.get_host_port_by_name(
topo_client, "test_keyspace.0.master:vts")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:vts", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
if shard_0_master_addrs[0][2] != True:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:vts" is not encrypted'
)
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(
topo_client, "test_keyspace.0.master:vt", encrypted=True)
if auto_addrs != shard_0_master_addrs:
self.fail(
'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
% (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection(
"%s:%u" %
(shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "",
"test_keyspace", "0", 10.0)
conn.dial()
self.fail("No exception raised to secure port")
except dbexceptions.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"",
"test_keyspace",
"0",
5.0,
encrypted=True)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
# start a vtgate to connect to that tablet
gate = utils.VtGate(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
gate.start(tablet_bson_encrypted=True)
# try to connect to vtgate with regular client
timeout = 2.0
try:
conn = vtgatev2.connect([
gate.secure_addr(),
], timeout)
self.fail("No exception raised to VTGate secure port")
except dbexceptions.OperationalError as e:
exception_type = e.args[2]
exception_msg = str(e.args[2][0][0])
self.assertIsInstance(exception_type, dbexceptions.FatalError,
"unexpected exception type")
if not exception_msg.startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception message: %s" % exception_msg)
sconn = utils.get_vars(gate.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to vtgate with encrypted port
conn = vtgatev2.connect([
gate.secure_addr(),
],
timeout,
encrypted=True)
(results, rowcount, lastrowid, fields) = conn._execute(
"select 1 from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(gate.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(gate.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a vtgate secure connection, see what exception we get
try:
conn._execute("select sleep(4) from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
conn.close()
gate.kill()
# kill everything
utils.vtgate.kill()
def test_cells_aliases(self):
utils.run_vtctl(['CreateKeyspace',
'--sharding_column_name', 'custom_ksid_col',
'--sharding_column_type', base_sharding.keyspace_id_type,
'test_keyspace'])
shard_0_master.init_tablet('replica', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_0_rdonly.init_tablet('rdonly', 'test_keyspace', '-80')
shard_1_master.init_tablet('replica', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
shard_1_rdonly.init_tablet('rdonly', 'test_keyspace', '80-')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
ks = utils.run_vtctl_json(['GetSrvKeyspace', 'test_nj', 'test_keyspace'])
self.assertEqual(ks['sharding_column_name'], 'custom_ksid_col')
# we set full_mycnf_args to True as a test in the KIT_BYTES case
full_mycnf_args = (base_sharding.keyspace_id_type ==
keyrange_constants.KIT_BYTES)
# create databases so vttablet can start behaving somewhat normally
for t in [shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica, shard_1_rdonly]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None, full_mycnf_args=full_mycnf_args,
binlog_use_v3_resharding_mode=False)
# wait for the tablets (replication is not setup, they won't be healthy)
for t in [shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica, shard_1_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias], auto_log=True)
# check the shards
shards = utils.run_vtctl_json(['FindAllShardsInKeyspace', 'test_keyspace'])
self.assertIn('-80', shards, 'unexpected shards: %s' % str(shards))
self.assertIn('80-', shards, 'unexpected shards: %s' % str(shards))
self.assertEqual(len(shards), 2, 'unexpected shards: %s' % str(shards))
# create the tables
self._create_schema()
self._insert_startup_values()
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# Make sure srv keyspace graph looks as expected
utils.check_srv_keyspace(
'test_nj', 'test_keyspace',
'Partitions(master): -80 80-\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
utils.check_srv_keyspace(
'test_ny', 'test_keyspace',
'Partitions(master): -80 80-\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# Bootstrap vtgate
utils.apply_vschema(vschema)
# Adds alias so vtgate can route to replica/rdonly tablets that are not in the same cell, but same alias
if use_alias:
utils.run_vtctl(['AddCellsAlias', '-cells', 'test_nj,test_ny','region_east_coast'], auto_log=True)
tablet_types_to_wait='MASTER,REPLICA'
else:
tablet_types_to_wait='MASTER'
utils.VtGate().start(
tablets=[shard_0_master, shard_1_master],
tablet_types_to_wait=tablet_types_to_wait,
cells_to_watch='test_nj,test_ny',
)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.master', 1)
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'master',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
if use_alias:
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'master',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'replica',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
vtgate_conn = self._get_connection()
result = self._execute_on_tablet_type(
vtgate_conn,
'rdonly',
'select count(*) from test_table', {})
self.assertEqual(
result,
([(3,)], 1, 0,
[('count(*)', self.int_type)]))
else:
vtgate_conn = self._get_connection()
try:
self._execute_on_tablet_type(
vtgate_conn,
'replica',
'select count(*) from test_table', {})
self.fail('Expected execute to fail, did not get error')
except Exception as e:
s = str(e)
self.assertIn('80.replica, no valid tablet: node', s)
vtgate_conn = self._get_connection()
try:
self._execute_on_tablet_type(
vtgate_conn,
'rdonly',
'select count(*) from test_table', {})
self.fail('Expected execute to fail, did not get error')
except Exception as e:
s = str(e)
self.assertIn('80.rdonly, no valid tablet: node', s)
def test_secure(self):
with open(table_acl_config, 'w') as fd:
fd.write("""{
"table_groups": [
{
"table_names_or_prefixes": ["vt_insert_test"],
"readers": ["vtgate client 1"],
"writers": ["vtgate client 1"],
"admins": ["vtgate client 1"]
}
]
}
""")
# start the tablets
shard_0_master.start_vttablet(
wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config,
extra_args=server_extra_args('vttablet-server-instance',
'vttablet-client'))
shard_0_slave.start_vttablet(
wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config,
extra_args=server_extra_args('vttablet-server-instance',
'vttablet-client'))
# setup replication
utils.run_vtctl(tmclient_extra_args('vttablet-client-1') + [
'InitShardMaster', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(tmclient_extra_args('vttablet-client-1') + [
'ApplySchema', '-sql', create_vt_insert_test,
'test_keyspace'])
for t in [shard_0_master, shard_0_slave]:
utils.run_vtctl(tmclient_extra_args('vttablet-client-1') + [
'RunHealthCheck', t.tablet_alias])
# start vtgate
utils.VtGate().start(extra_args=tabletconn_extra_args('vttablet-client-1')+
server_extra_args('vtgate-server-instance',
'vtgate-client'))
# 'vtgate client 1' is authorized to access vt_insert_test
protocol, addr = utils.vtgate.rpc_endpoint(python=True)
conn = vtgate_client.connect(protocol, addr, 30.0,
**python_client_kwargs('vtgate-client-1',
'vtgate-server'))
cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace',
shards=['0'])
cursor.execute('select * from vt_insert_test', {})
conn.close()
# 'vtgate client 2' is not authorized to access vt_insert_test
conn = vtgate_client.connect(protocol, addr, 30.0,
**python_client_kwargs('vtgate-client-2',
'vtgate-server'))
try:
cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace',
shards=['0'])
cursor.execute('select * from vt_insert_test', {})
self.fail('Execute went through')
except dbexceptions.DatabaseError, e:
s = str(e)
self.assertIn('table acl error', s)
self.assertIn('cannot run PASS_SELECT on table', s)
def setUp(self):
self.master = tablet.Tablet()
self.replica = tablet.Tablet()
self.all_tablets = [self.master, self.replica]
try:
environment.topo_server().setup()
setup_procs = [t.init_mysql() for t in self.all_tablets]
utils.Vtctld().start()
utils.wait_procs(setup_procs)
utils.run_vtctl(['CreateKeyspace', KEYSPACE])
# Start tablets.
db_name = 'vt_' + KEYSPACE
for t in self.all_tablets:
t.create_db(db_name)
self.master.start_vttablet(wait_for_state=None,
init_tablet_type='replica',
init_keyspace=KEYSPACE,
init_shard=SHARD,
tablet_index=0)
self.replica.start_vttablet(wait_for_state=None,
init_tablet_type='replica',
init_keyspace=KEYSPACE,
init_shard=SHARD,
tablet_index=1)
for t in self.all_tablets:
t.wait_for_vttablet_state('NOT_SERVING')
# Reparent to choose an initial master and enable replication.
utils.run_vtctl([
'InitShardMaster', '-force',
'%s/%s' % (KEYSPACE, SHARD), self.master.tablet_alias
])
# Create the schema.
utils.run_vtctl(['ApplySchema', '-sql=' + SCHEMA, KEYSPACE])
# Start vtgate.
utils.VtGate().start(
extra_args=[
'-enable_vtgate_buffer',
# Long timeout in case failover is slow.
'-vtgate_buffer_window',
'10m',
'-vtgate_buffer_max_failover_duration',
'10m',
'-vtgate_buffer_min_time_between_failovers',
'20m'
],
tablets=self.all_tablets)
# Insert two rows for the later threads (critical read, update).
with utils.vtgate.write_transaction(keyspace=KEYSPACE,
shards=[SHARD],
tablet_type='master') as tx:
tx.execute('INSERT INTO buffer (id, msg) VALUES (:id, :msg)', {
'id': CRITICAL_READ_ROW_ID,
'msg': 'critical read'
})
tx.execute('INSERT INTO buffer (id, msg) VALUES (:id, :msg)', {
'id': UPDATE_ROW_ID,
'msg': 'update'
})
except:
self.tearDown()
raise
def setUpModule():
global master_start_position
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# start a vtctld so the vtctl insert commands are just RPCs, not forks
utils.Vtctld().start()
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master',
'test_keyspace',
'0',
tablet_index=0)
replica_tablet.init_tablet('replica',
'test_keyspace',
'0',
tablet_index=1)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
master_tablet.create_db('other_database')
replica_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('other_database')
master_tablet.start_vttablet(wait_for_state=None)
replica_tablet.start_vttablet(wait_for_state=None)
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('NOT_SERVING')
for t in [master_tablet, replica_tablet]:
t.reset_replication()
utils.run_vtctl(
['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias],
auto_log=True)
utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica')
master_tablet.wait_for_vttablet_state('SERVING')
replica_tablet.wait_for_vttablet_state('SERVING')
# reset counter so tests don't assert
tablet.Tablet.tablets_running = 0
master_start_position = _get_master_current_position()
master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
master_tablet.mquery('vt_test_keyspace', _create_vt_a)
master_tablet.mquery('vt_test_keyspace', _create_vt_b)
utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
utils.run_vtctl(['RebuildVSchemaGraph'])
utils.VtGate().start(tablets=[master_tablet, replica_tablet])
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
# Wait for the master and slave tablet's ReloadSchema to have worked.
# Note we don't specify a keyspace name, there is only one, vschema
# will just use that single keyspace.
timeout = 10
while True:
try:
utils.vtgate.execute('select count(1) from vt_insert_test',
tablet_type='master')
utils.vtgate.execute('select count(1) from vt_insert_test',
tablet_type='replica')
break
except protocols_flavor().client_error_exception_type():
logging.exception('query failed')
timeout = utils.wait_step('slave tablet having correct schema',
timeout)
# also re-run ReloadSchema on slave, it case the first one
# didn't get the replicated table.
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
except:
tearDownModule()
raise
def setUpModule():
global master_start_position
try:
environment.topo_server().setup()
# start mysql instance external to the test
setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
utils.wait_procs(setup_procs)
# start a vtctld so the vtctl insert commands are just RPCs, not forks
utils.Vtctld().start()
# Start up a master mysql and vttablet
logging.debug('Setting up tablets')
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
master_tablet.init_tablet('master', 'test_keyspace', '0')
replica_tablet.init_tablet('replica', 'test_keyspace', '0')
utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
utils.validate_topology()
master_tablet.create_db('vt_test_keyspace')
master_tablet.create_db('other_database')
replica_tablet.create_db('vt_test_keyspace')
replica_tablet.create_db('other_database')
utils.VtGate().start()
master_tablet.start_vttablet()
replica_tablet.start_vttablet()
utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
utils.check_db_read_write(master_tablet.tablet_uid)
for t in [master_tablet, replica_tablet]:
t.reset_replication()
utils.run_vtctl(
['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias],
auto_log=True)
# reset counter so tests don't assert
tablet.Tablet.tablets_running = 0
master_start_position = _get_master_current_position()
master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
master_tablet.mquery('vt_test_keyspace', _create_vt_a)
master_tablet.mquery('vt_test_keyspace', _create_vt_b)
utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
# wait for the master and slave tablet's ReloadSchema to have worked
timeout = 10
while True:
try:
master_tablet.execute('select count(1) from vt_insert_test')
replica_tablet.execute('select count(1) from vt_insert_test')
break
except protocols_flavor().client_error_exception_type():
logging.exception('query failed')
timeout = utils.wait_step('slave tablet having correct schema',
timeout)
# also re-run ReloadSchema on slave, it case the first one
# didn't get the replicated table.
utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
except:
tearDownModule()
raise
def test_mysql_connector(self):
with open(table_acl_config, 'w') as fd:
fd.write("""{
"table_groups": [
{
"table_names_or_prefixes": ["vt_insert_test", "dual"],
"readers": ["vtgate client 1"],
"writers": ["vtgate client 1"],
"admins": ["vtgate client 1"]
}
]
}
""")
with open(mysql_auth_server_static, 'w') as fd:
fd.write("""{
"testuser1": {
"Password": "******",
"UserData": "vtgate client 1"
},
"testuser2": {
"Password": "******",
"UserData": "vtgate client 2"
}
}
""")
# start the tablets
shard_0_master.start_vttablet(wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config)
shard_0_slave.start_vttablet(wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config)
# setup replication
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(
['ApplySchema', '-sql', create_vt_insert_test, 'test_keyspace'])
for t in [shard_0_master, shard_0_slave]:
utils.run_vtctl(['RunHealthCheck', t.tablet_alias])
# start vtgate
utils.VtGate(mysql_server=True).start(extra_args=[
'-mysql_auth_server_impl', 'static', '-mysql_server_query_timeout',
'1s', '-mysql_auth_server_static_file', mysql_auth_server_static
])
# We use gethostbyname('localhost') so we don't presume
# of the IP format (travis is only IP v4, really).
params = dict(host=socket.gethostbyname('localhost'),
port=utils.vtgate.mysql_port,
user='******',
passwd='testpassword1',
db='test_keyspace')
# 'vtgate client 1' is authorized to access vt_insert_test
conn = MySQLdb.Connect(**params)
cursor = conn.cursor()
cursor.execute('select * from vt_insert_test', {})
cursor.close()
# Test multi-statement support. It should only work when
# COM_SET_OPTION has set the options to 0
conn.set_server_option(self.MYSQL_OPTION_MULTI_STATEMENTS_ON)
cursor = conn.cursor()
cursor.execute("select 1; select 2")
self.assertEquals(((1L, ), ), cursor.fetchall())
self.assertEquals(1, cursor.nextset())
self.assertEquals(((2L, ), ), cursor.fetchall())
self.assertEquals(None, cursor.nextset())
cursor.close()
conn.set_server_option(self.MYSQL_OPTION_MULTI_STATEMENTS_OFF)
# Multi-statement support should not work without the
# option enabled
cursor = conn.cursor()
try:
cursor.execute("select 1; select 2")
self.fail('Execute went through')
except MySQLdb.OperationalError, e:
s = str(e)
self.assertIn('syntax error', s)
def test_prepared_statements(self):
with open(table_acl_config, 'w') as fd:
fd.write("""{
"table_groups": [
{
"table_names_or_prefixes": ["vt_prepare_stmt_test", "dual"],
"readers": ["vtgate client 1"],
"writers": ["vtgate client 1"],
"admins": ["vtgate client 1"]
}
]
}
""")
with open(mysql_auth_server_static, 'w') as fd:
fd.write("""{
"testuser1": {
"Password": "******",
"UserData": "vtgate client 1"
},
"testuser2": {
"Password": "******",
"UserData": "vtgate client 2"
}
}
""")
# start the tablets
shard_0_master.start_vttablet(wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config)
shard_0_slave.start_vttablet(wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config)
# setup replication
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(
['ApplySchema', '-sql', create_vt_prepare_test, 'test_keyspace'])
for t in [shard_0_master, shard_0_slave]:
utils.run_vtctl(['RunHealthCheck', t.tablet_alias])
# start vtgate
utils.VtGate(mysql_server=True).start(extra_args=[
'-mysql_auth_server_impl', 'static', '-mysql_server_query_timeout',
'1s', '-mysql_auth_server_static_file', mysql_auth_server_static
])
# We use gethostbyname('localhost') so we don't presume
# of the IP format (travis is only IP v4, really).
params = dict(host=socket.gethostbyname('localhost'),
port=utils.vtgate.mysql_port,
user='******',
passwd='testpassword1',
db='test_keyspace',
use_pure=True)
# 'vtgate client 1' is authorized to access vt_prepare_insert_test
conn = mysql.connector.Connect(**params)
cursor = conn.cursor()
cursor.execute('select * from vt_prepare_stmt_test', {})
cursor.fetchone()
cursor.close()
cursor = conn.cursor()
try:
cursor.execute('selet * from vt_prepare_stmt_test', {})
cursor.close()
except mysql.connector.Error as err:
if err.errno == 1105:
print "Captured the error"
else:
raise
# Insert several rows using prepared statements
text_value = "text" * 100 # Large text value
largeComment = 'L' * ((4 * 1024) + 1) # Large blob
# Set up the values for the prepared statement
cursor = conn.cursor(cursor_class=MySQLCursorPrepared)
for i in range(1, 100):
insert_values = (i, str(i) + "21", i * 100, 127, 1, 32767, 8388607,
2147483647, 2.55, 64.9, 55.5,
datetime.date(2009, 5,
5), datetime.date(2009, 5, 5),
datetime.datetime.now().time(),
datetime.date(2009, 5, 5), 1, 1, 1, 1, 1, 1, 1, 1,
1, json_example, text_value, largeComment)
cursor.execute(insert_stmt, insert_values)
cursor.fetchone()
cursor.close()
cursor = conn.cursor(cursor_class=MySQLCursorPrepared)
cursor.execute('select * from vt_prepare_stmt_test where id = %s',
(1, ))
result = cursor.fetchall()
# Validate the query results.
if cursor.rowcount != 1:
self.fail('expected 1 row got ' + str(cursor.rowcount))
if result[0][1] != "121":
self.fail('Received incorrect value, wanted: 121, got ' +
result[1])
cursor.close()
# Update a row using prepared statements
updated_text_value = "text_col_msg"
updated_data_value = "updated"
cursor = conn.cursor(cursor_class=MySQLCursorPrepared)
cursor.execute(
'update vt_prepare_stmt_test set data = %s , text_col = %s where id = %s',
(updated_data_value, updated_text_value, 1))
cursor.close()
# Validate the update results
cursor = conn.cursor(cursor_class=MySQLCursorPrepared)
cursor.execute('select * from vt_prepare_stmt_test where id = %s',
(1, ))
result = cursor.fetchone()
if result[-1] != updated_data_value or result[-2] != updated_text_value:
self.fail("Received incorrect values")
cursor.close()
# Delete from table using prepared statements
cursor = conn.cursor(cursor_class=MySQLCursorPrepared)
cursor.execute('delete from vt_prepare_stmt_test where text_col = %s',
(text_value, ))
cursor.close()
# Validate Deletion
cursor = conn.cursor(cursor_class=MySQLCursorPrepared)
cursor.execute('select count(*) from vt_prepare_stmt_test')
res = cursor.fetchone()
if res[0] != 1:
self.fail("Delete failed")
cursor.close()
def restart_vtgate(extra_args=None):
if extra_args is None:
extra_args = {}
port = utils.vtgate.port
utils.vtgate.kill()
utils.VtGate(port=port).start(extra_args=extra_args)
def restart_vtgate(extra_args={}):
port = utils.vtgate.port
utils.vtgate.kill()
utils.VtGate(port=port).start(extra_args=extra_args)
def test_sharded_recovery(self):
"""Test recovery from backup flow.
test_recovery will:
- create a shard with master and replica1 only
- run InitShardMaster
- insert some data
- perform a resharding
- take a backup of both new shards
- insert more data on the masters of both shards
- create a recovery keyspace
- bring up tablet_replica2 and tablet_replica3 in the new keyspace
- check that new tablets do not have data created after backup
- check that vtgate queries work correctly
"""
# insert data on master, wait for replica to get it
utils.run_vtctl([
'ApplySchema', '-sql', self._create_vt_insert_test, 'test_keyspace'
],
auto_log=True)
self._insert_data(tablet_master, 1)
self._check_data(tablet_replica1, 1, 'replica1 tablet getting data')
# insert more data on the master
self._insert_data(tablet_master, 4)
utils.run_vtctl(
['ApplyVSchema', '-vschema', self._vschema_json, 'test_keyspace'],
auto_log=True)
# create the split shards
shard_0_master.init_tablet('replica',
keyspace='test_keyspace',
shard='-80',
tablet_index=0)
shard_0_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='-80',
tablet_index=1)
shard_0_rdonly.init_tablet('rdonly',
keyspace='test_keyspace',
shard='-80',
tablet_index=2)
shard_1_master.init_tablet('replica',
keyspace='test_keyspace',
shard='80-',
tablet_index=0)
shard_1_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='80-',
tablet_index=1)
shard_1_rdonly.init_tablet('rdonly',
keyspace='test_keyspace',
shard='80-',
tablet_index=2)
for t in [
shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica, shard_1_rdonly
]:
t.start_vttablet(wait_for_state=None,
binlog_use_v3_resharding_mode=True)
for t in [
shard_0_master, shard_0_replica, shard_0_rdonly,
shard_1_master, shard_1_replica, shard_1_rdonly
]:
t.wait_for_vttablet_state('NOT_SERVING')
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias
],
auto_log=True)
for t in [shard_0_replica, shard_1_replica]:
utils.wait_for_tablet_type(t.tablet_alias, 'replica')
sharded_tablets = [
shard_0_master, shard_0_replica, shard_0_rdonly, shard_1_master,
shard_1_replica, shard_1_rdonly
]
for t in sharded_tablets:
t.wait_for_vttablet_state('SERVING')
# we need to create the schema, and the worker will do data copying
for keyspace_shard in ('test_keyspace/-80', 'test_keyspace/80-'):
utils.run_vtctl(
['CopySchemaShard', 'test_keyspace/0', keyspace_shard],
auto_log=True)
utils.run_vtctl(['SplitClone', 'test_keyspace', '0', '-80,80-'],
auto_log=True)
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'rdonly'],
auto_log=True)
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'replica'],
auto_log=True)
# then serve master from the split shards
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'master'],
auto_log=True)
# remove the original tablets in the original shard
tablet.kill_tablets([tablet_master, tablet_replica1, tablet_rdonly])
for t in [tablet_replica1, tablet_rdonly]:
utils.run_vtctl(['DeleteTablet', t.tablet_alias], auto_log=True)
utils.run_vtctl(
['DeleteTablet', '-allow_master', tablet_master.tablet_alias],
auto_log=True)
# rebuild the serving graph, all mentions of the old shards should be gone
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# delete the original shard
utils.run_vtctl(['DeleteShard', 'test_keyspace/0'], auto_log=True)
result = shard_0_master.mquery('vt_test_keyspace',
"select count(*) from vt_insert_test")
shard_0_count = result[0][0]
logging.debug("Shard -80 has %d rows", shard_0_count)
shard_0_test_id = 0
if shard_0_count > 0:
result = shard_0_master.mquery('vt_test_keyspace',
"select id from vt_insert_test")
shard_0_test_id = result[0][0]
result = shard_1_master.mquery('vt_test_keyspace',
"select count(*) from vt_insert_test")
shard_1_count = result[0][0]
logging.debug("Shard 80- has %d rows", shard_1_count)
shard_1_test_id = 0
if shard_1_count > 0:
result = shard_1_master.mquery('vt_test_keyspace',
"select id from vt_insert_test")
shard_1_test_id = result[0][0]
# backup the new shards
utils.run_vtctl(['Backup', shard_0_replica.tablet_alias],
auto_log=True)
utils.run_vtctl(['Backup', shard_1_replica.tablet_alias],
auto_log=True)
# check that the backup shows up in the listing
backups = self._list_backups('-80')
logging.debug('list of backups: %s', backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].endswith(shard_0_replica.tablet_alias))
backups = self._list_backups('80-')
logging.debug('list of backups: %s', backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].endswith(shard_1_replica.tablet_alias))
# start vtgate
vtgate = utils.VtGate()
vtgate.start(tablets=[shard_0_master, shard_1_master],
tablet_types_to_wait='MASTER')
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.master', 1)
vtgate_conn = get_connection()
cursor = vtgate_conn.cursor(tablet_type='master',
keyspace=None,
writable=True)
# insert more data on the masters
for i in [2, 3]:
cursor.execute(
'insert into vt_insert_test (id, msg) values (:id, :msg)', {
'id': i,
'msg': 'test %s' % i
})
vtgate_conn.close()
vtgate.kill()
# now bring up the recovery keyspace and 2 tablets, letting it restore from backup.
self._restore(tablet_replica2, 'recovery_keyspace', '-80')
self._restore(tablet_replica3, 'recovery_keyspace', '80-')
# check the new replicas have the correct number of rows
self._check_data(tablet_replica2, shard_0_count,
'replica2 tablet should not have new data')
self._check_data(tablet_replica3, shard_1_count,
'replica3 tablet should not have new data')
# start vtgate
vtgate = utils.VtGate()
vtgate.start(tablets=[
shard_0_master, shard_0_replica, shard_1_master, shard_1_replica,
tablet_replica2, tablet_replica3
],
tablet_types_to_wait='REPLICA')
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.replica', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.replica', 1)
utils.vtgate.wait_for_endpoints('recovery_keyspace.-80.replica', 1)
utils.vtgate.wait_for_endpoints('recovery_keyspace.80-.replica', 1)
# check that vtgate doesn't route queries to new tablet
vtgate_conn = get_connection()
cursor = vtgate_conn.cursor(tablet_type='replica',
keyspace=None,
writable=True)
cursor.execute('select count(*) from vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 4)
# check that new keyspace is accessible by using ks.table
cursor.execute('select count(*) from recovery_keyspace.vt_insert_test',
{})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 2)
# check that new keyspace is accessible with 'use ks'
cursor.execute('use recovery_keyspace@replica', {})
cursor.execute('select count(*) from vt_insert_test', {})
result = cursor.fetchall()
if not result:
self.fail('Result cannot be null')
else:
self.assertEqual(result[0][0], 2)
# TODO check that new tablet is accessible with 'use ks:shard'
# this currently does not work through the python client, though it works from mysql client
#cursor.execute('use recovery_keyspace:-80@replica', {})
#cursor.execute('select count(*) from vt_insert_test', {})
#result = cursor.fetchall()
#if not result:
# self.fail('Result cannot be null')
#else:
# self.assertEqual(result[0][0], shard_0_count)
#cursor.execute('select id from vt_insert_test', {})
#result = cursor.fetchall()
#if not result:
# self.fail('Result cannot be null')
#else:
# self.assertEqual(result[0][0], shard_0_test_id)
#cursor.execute('use recovery_keyspace:80-@replica', {})
#cursor.execute('select count(*) from vt_insert_test', {})
#result = cursor.fetchall()
#if not result:
# self.fail('Result cannot be null')
#else:
# self.assertEqual(result[0][0], shard_1_count)
#cursor.execute('use recovery_keyspace:80-@replica', {})
#cursor.execute('select id from vt_insert_test', {})
#result = cursor.fetchall()
#if not result:
# self.fail('Result cannot be null')
#else:
# self.assertEqual(result[0][0], shard_1_test_id)
vtgate_conn.close()
tablet_replica2.kill_vttablet()
tablet_replica3.kill_vttablet()
vtgate.kill()
def test_resharding(self):
# create the keyspace with just one shard
shard_master.init_tablet('master',
keyspace='test_keyspace',
shard='0',
tablet_index=0)
shard_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='0',
tablet_index=1)
shard_rdonly1.init_tablet('rdonly',
keyspace='test_keyspace',
shard='0',
tablet_index=2)
for t in [shard_master, shard_replica, shard_rdonly1]:
t.create_db('vt_test_keyspace')
shard_master.start_vttablet(wait_for_state=None)
shard_replica.start_vttablet(wait_for_state=None)
shard_rdonly1.start_vttablet(wait_for_state=None)
shard_master.wait_for_vttablet_state('SERVING')
for t in [shard_replica, shard_rdonly1]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
shard_master.tablet_alias
],
auto_log=True)
utils.wait_for_tablet_type(shard_replica.tablet_alias, 'replica')
utils.wait_for_tablet_type(shard_rdonly1.tablet_alias, 'rdonly')
for t in [shard_master, shard_replica, shard_rdonly1]:
t.wait_for_vttablet_state('SERVING')
# create the tables and add startup values
self._create_schema()
self._insert_startup_values()
# reload schema on all tablets so we can query them
for t in [shard_master, shard_replica, shard_rdonly1]:
utils.run_vtctl(['ReloadSchema', t.tablet_alias], auto_log=True)
# must start vtgate after tablets are up, or else wait until 1min refresh
# we want cache_ttl at zero so we re-read the topology for every test query.
utils.VtGate().start(
cache_ttl='0',
tablets=[shard_master, shard_replica, shard_rdonly1])
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1)
# check the Map Reduce API works correctly, should use ExecuteShards,
# as we're not sharded yet.
# we have 3 values in the database, asking for 4 splits will get us
# a single query.
sql = 'select id, msg from resharding1'
s = utils.vtgate.split_query(sql, 'test_keyspace', 4)
self.assertEqual(len(s), 1)
self.assertEqual(s[0]['shard_part']['shards'][0], '0')
# change the schema, backfill keyspace_id, and change schema again
self._add_sharding_key_to_schema()
self._backfill_keyspace_id(shard_master)
self._mark_sharding_key_not_null()
# now we can be a sharded keyspace (and propagate to SrvKeyspace)
utils.run_vtctl([
'SetKeyspaceShardingInfo', 'test_keyspace', 'keyspace_id',
keyspace_id_type
])
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# run a health check on source replica so it responds to discovery
utils.run_vtctl(['RunHealthCheck', shard_replica.tablet_alias])
# create the split shards
shard_0_master.init_tablet('master',
keyspace='test_keyspace',
shard='-80',
tablet_index=0)
shard_0_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='-80',
tablet_index=1)
shard_0_rdonly1.init_tablet('rdonly',
keyspace='test_keyspace',
shard='-80',
tablet_index=2)
shard_1_master.init_tablet('master',
keyspace='test_keyspace',
shard='80-',
tablet_index=0)
shard_1_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='80-',
tablet_index=1)
shard_1_rdonly1.init_tablet('rdonly',
keyspace='test_keyspace',
shard='80-',
tablet_index=2)
for t in [
shard_0_master, shard_0_replica, shard_1_master,
shard_1_replica
]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [shard_0_rdonly1, shard_1_rdonly1]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [shard_0_master, shard_1_master]:
t.wait_for_vttablet_state('SERVING')
for t in [
shard_0_replica, shard_0_rdonly1, shard_1_replica,
shard_1_rdonly1
]:
t.wait_for_vttablet_state('NOT_SERVING')
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias
],
auto_log=True)
for t in [shard_0_replica, shard_1_replica]:
utils.wait_for_tablet_type(t.tablet_alias, 'replica')
for t in [shard_0_rdonly1, shard_1_rdonly1]:
utils.wait_for_tablet_type(t.tablet_alias, 'rdonly')
sharded_tablets = [
shard_0_master, shard_0_replica, shard_0_rdonly1, shard_1_master,
shard_1_replica, shard_1_rdonly1
]
for t in sharded_tablets:
t.wait_for_vttablet_state('SERVING')
# must restart vtgate after tablets are up, or else wait until 1min refresh
# we want cache_ttl at zero so we re-read the topology for every test query.
utils.vtgate.kill()
utils.VtGate().start(cache_ttl='0',
tablets=[
shard_master, shard_replica, shard_rdonly1,
shard_0_master, shard_0_replica,
shard_0_rdonly1, shard_1_master,
shard_1_replica, shard_1_rdonly1
])
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.replica', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.replica', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
# check the Map Reduce API works correctly, should use ExecuteKeyRanges now,
# as we are sharded (with just one shard).
# again, we have 3 values in the database, asking for 4 splits will get us
# a single query.
sql = 'select id, msg from resharding1'
s = utils.vtgate.split_query(sql, 'test_keyspace', 4)
self.assertEqual(len(s), 1)
self.assertEqual(s[0]['key_range_part']['keyspace'], 'test_keyspace')
# There must be one empty KeyRange which represents the full keyspace.
self.assertEqual(len(s[0]['key_range_part']['key_ranges']), 1)
self.assertEqual(s[0]['key_range_part']['key_ranges'][0], {})
utils.check_srv_keyspace('test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -\n'
'Partitions(replica): -\n',
keyspace_id_type=keyspace_id_type)
# we need to create the schema, and the worker will do data copying
for keyspace_shard in ('test_keyspace/-80', 'test_keyspace/80-'):
utils.run_vtctl([
'CopySchemaShard', '--exclude_tables', 'unrelated',
shard_rdonly1.tablet_alias, keyspace_shard
],
auto_log=True)
utils.run_vtctl(['RunHealthCheck', shard_rdonly1.tablet_alias])
utils.run_vtworker([
'--cell', 'test_nj', '--command_display_interval', '10ms',
'SplitClone', '--exclude_tables', 'unrelated',
'--source_reader_count', '10', '--min_table_size_for_split', '1',
'--min_healthy_rdonly_tablets', '1', 'test_keyspace/0'
],
auto_log=True)
# check the startup values are in the right place
self._check_startup_values()
# check the schema too
utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
auto_log=True)
# check the binlog players are running
logging.debug('Waiting for binlog players to start on new masters...')
self.check_destination_master(shard_0_master, ['test_keyspace/0'])
self.check_destination_master(shard_1_master, ['test_keyspace/0'])
# check that binlog server exported the stats vars
self.check_binlog_server_vars(shard_replica, horizontal=True)
# testing filtered replication: insert a bunch of data on shard 1,
# check we get most of it after a few seconds, wait for binlog server
# timeout, check we get all of it.
logging.debug('Inserting lots of data on source shard')
self._insert_lots(1000)
logging.debug('Checking 80 percent of data is sent quickly')
v = self._check_lots_timeout(1000, 80, 5)
if v != 100:
logging.debug('Checking all data goes through eventually')
self._check_lots_timeout(1000, 100, 20)
logging.debug('Checking no data was sent the wrong way')
self._check_lots_not_present(1000)
self.check_binlog_player_vars(shard_0_master, ['test_keyspace/0'],
seconds_behind_master_max=30)
self.check_binlog_player_vars(shard_1_master, ['test_keyspace/0'],
seconds_behind_master_max=30)
self.check_binlog_server_vars(shard_replica,
horizontal=True,
min_statements=1000,
min_transactions=1000)
# use vtworker to compare the data
logging.debug('Running vtworker SplitDiff for -80')
for t in [shard_0_rdonly1, shard_1_rdonly1]:
utils.run_vtctl(['RunHealthCheck', t.tablet_alias])
utils.run_vtworker([
'-cell', 'test_nj', 'SplitDiff', '--min_healthy_rdonly_tablets',
'1', 'test_keyspace/-80'
],
auto_log=True)
logging.debug('Running vtworker SplitDiff for 80-')
utils.run_vtworker([
'-cell', 'test_nj', 'SplitDiff', '--min_healthy_rdonly_tablets',
'1', 'test_keyspace/80-'
],
auto_log=True)
utils.pause('Good time to test vtworker for diffs')
# get status for the destination master tablet, make sure we have it all
self.check_running_binlog_player(shard_0_master, 2000, 2000)
self.check_running_binlog_player(shard_1_master, 6000, 2000)
# check we can't migrate the master just yet
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'master'],
expect_fail=True)
# now serve rdonly from the split shards
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'rdonly'],
auto_log=True)
utils.check_srv_keyspace('test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -\n',
keyspace_id_type=keyspace_id_type)
# make sure rdonly tablets are back to serving before hitting vtgate.
for t in [shard_0_rdonly1, shard_1_rdonly1]:
t.wait_for_vttablet_state('SERVING')
utils.vtgate.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
# check the Map Reduce API works correctly, should use ExecuteKeyRanges
# on both destination shards now.
# we ask for 2 splits to only have one per shard
sql = 'select id, msg from resharding1'
s = utils.vtgate.split_query(sql, 'test_keyspace', 2)
self.assertEqual(len(s), 2)
self.assertEqual(s[0]['key_range_part']['keyspace'], 'test_keyspace')
self.assertEqual(s[1]['key_range_part']['keyspace'], 'test_keyspace')
self.assertEqual(len(s[0]['key_range_part']['key_ranges']), 1)
self.assertEqual(len(s[1]['key_range_part']['key_ranges']), 1)
# then serve replica from the split shards
source_tablet = shard_replica
destination_tablets = [shard_0_replica, shard_1_replica]
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'replica'],
auto_log=True)
utils.check_srv_keyspace('test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=keyspace_id_type)
# move replica back and forth
utils.run_vtctl(
['MigrateServedTypes', '-reverse', 'test_keyspace/0', 'replica'],
auto_log=True)
# After a backwards migration, queryservice should be enabled on
# source and disabled on destinations
utils.check_tablet_query_service(self, source_tablet, True, False)
utils.check_tablet_query_services(self, destination_tablets, False,
True)
utils.check_srv_keyspace('test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -\n',
keyspace_id_type=keyspace_id_type)
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'replica'],
auto_log=True)
# After a forwards migration, queryservice should be disabled on
# source and enabled on destinations
utils.check_tablet_query_service(self, source_tablet, False, True)
utils.check_tablet_query_services(self, destination_tablets, True,
False)
utils.check_srv_keyspace('test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=keyspace_id_type)
# then serve master from the split shards
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'master'],
auto_log=True)
utils.check_srv_keyspace('test_nj',
'test_keyspace',
'Partitions(master): -80 80-\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=keyspace_id_type)
# check the binlog players are gone now
self.check_no_binlog_player(shard_0_master)
self.check_no_binlog_player(shard_1_master)
# make sure we can't delete a shard with tablets
utils.run_vtctl(['DeleteShard', 'test_keyspace/0'], expect_fail=True)
# remove the original tablets in the original shard
tablet.kill_tablets([shard_master, shard_replica, shard_rdonly1])
for t in [shard_replica, shard_rdonly1]:
utils.run_vtctl(['DeleteTablet', t.tablet_alias], auto_log=True)
utils.run_vtctl(
['DeleteTablet', '-allow_master', shard_master.tablet_alias],
auto_log=True)
# rebuild the serving graph, all mentions of the old shards shoud be gone
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# delete the original shard
utils.run_vtctl(['DeleteShard', 'test_keyspace/0'], auto_log=True)
# kill everything else
tablet.kill_tablets([
shard_0_master, shard_0_replica, shard_0_rdonly1, shard_1_master,
shard_1_replica, shard_1_rdonly1
])
def test_mysql_connector(self):
with open(table_acl_config, 'w') as fd:
fd.write("""{
"table_groups": [
{
"table_names_or_prefixes": ["vt_insert_test", "dual"],
"readers": ["vtgate client 1"],
"writers": ["vtgate client 1"],
"admins": ["vtgate client 1"]
}
]
}
""")
with open(mysql_auth_server_static, 'w') as fd:
fd.write("""{
"testuser1": {
"Password": "******",
"UserData": "vtgate client 1"
},
"testuser2": {
"Password": "******",
"UserData": "vtgate client 2"
}
}
""")
# start the tablets
shard_0_master.start_vttablet(wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config)
shard_0_slave.start_vttablet(wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config)
# setup replication
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(
['ApplySchema', '-sql', create_vt_insert_test, 'test_keyspace'])
for t in [shard_0_master, shard_0_slave]:
utils.run_vtctl(['RunHealthCheck', t.tablet_alias])
# start vtgate
utils.VtGate(mysql_server=True).start(extra_args=[
'-mysql_auth_server_impl', 'static', '-mysql_server_query_timeout',
'1s', '-mysql_auth_server_static_file', mysql_auth_server_static
])
# We use gethostbyname('localhost') so we don't presume
# of the IP format (travis is only IP v4, really).
params = dict(host=socket.gethostbyname('localhost'),
port=utils.vtgate.mysql_port,
user='******',
passwd='testpassword1',
db='test_keyspace')
# 'vtgate client 1' is authorized to access vt_insert_test
conn = MySQLdb.Connect(**params)
cursor = conn.cursor()
cursor.execute('select * from vt_insert_test', {})
cursor.close()
# verify that queries work end-to-end with large grpc messages
largeComment = 'L' * ((4 * 1024 * 1024) + 1)
cursor = conn.cursor()
cursor.execute(
'insert into vt_insert_test (id, msg, keyspace_id, data) values(%s, %s, %s, %s) /* %s */',
(1, 'large blob', 123, 'LLL', largeComment))
cursor.close()
cursor = conn.cursor()
cursor.execute('select * from vt_insert_test where id = 1')
if cursor.rowcount != 1:
self.fail('expected 1 row got ' + str(cursor.rowcount))
for (id, msg, keyspace_id, blob) in cursor:
if blob != 'LLL':
self.fail('blob did not match \'LLL\'')
cursor.close()
hugeBlob = 'L' * (environment.grpc_max_message_size + 1)
cursor = conn.cursor()
try:
cursor.execute(
'insert into vt_insert_test (id, msg, keyspace_id, data) values(%s, %s, %s, %s)',
(2, 'huge blob', 123, hugeBlob))
self.fail('Execute went through')
except MySQLdb.OperationalError, e:
s = str(e)
self.assertIn('trying to send message larger than max', s)
def setup_tablets(): setup_sharded_keyspace() setup_unsharded_keyspace() utils.VtGate().start()
def test_custom_end_to_end(self):
"""Runs through the common operations of a custom sharded keyspace.
Tests creation with one shard, schema change, reading / writing
data, adding one more shard, reading / writing data from both
shards, applying schema changes again, and reading / writing data
from both shards again.
"""
# start the first shard only for now
shard_0_master.start_vttablet(
wait_for_state=None, target_tablet_type='replica',
init_keyspace='test_keyspace', init_shard='0')
shard_0_replica.start_vttablet(
wait_for_state=None, target_tablet_type='replica',
init_keyspace='test_keyspace', init_shard='0')
shard_0_rdonly.start_vttablet(
wait_for_state=None, target_tablet_type='rdonly',
init_keyspace='test_keyspace', init_shard='0')
for t in [shard_0_master, shard_0_replica, shard_0_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias], auto_log=True)
for t in [shard_0_master, shard_0_replica, shard_0_rdonly]:
t.wait_for_vttablet_state('SERVING')
self._check_shards_count_in_srv_keyspace(1)
s = utils.run_vtctl_json(['GetShard', 'test_keyspace/0'])
self.assertEqual(len(s['served_types']), 3)
# create a table on shard 0
sql = '''create table data(
id bigint auto_increment,
name varchar(64),
primary key (id)
) Engine=InnoDB'''
utils.run_vtctl(['ApplySchema', '-sql=' + sql, 'test_keyspace'],
auto_log=True)
# reload schema everywhere so the QueryService knows about the tables
for t in [shard_0_master, shard_0_replica, shard_0_rdonly]:
utils.run_vtctl(['ReloadSchema', t.tablet_alias], auto_log=True)
# create shard 1
shard_1_master.start_vttablet(
wait_for_state=None, target_tablet_type='replica',
init_keyspace='test_keyspace', init_shard='1')
shard_1_replica.start_vttablet(
wait_for_state=None, target_tablet_type='replica',
init_keyspace='test_keyspace', init_shard='1')
shard_1_rdonly.start_vttablet(
wait_for_state=None, target_tablet_type='rdonly',
init_keyspace='test_keyspace', init_shard='1')
for t in [shard_1_master, shard_1_replica, shard_1_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
s = utils.run_vtctl_json(['GetShard', 'test_keyspace/1'])
self.assertEqual(len(s['served_types']), 3)
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/1',
shard_1_master.tablet_alias], auto_log=True)
for t in [shard_1_master, shard_1_replica, shard_1_rdonly]:
t.wait_for_vttablet_state('SERVING')
utils.run_vtctl(['CopySchemaShard', shard_0_rdonly.tablet_alias,
'test_keyspace/1'], auto_log=True)
# must start vtgate after tablets are up, or else wait until 1min refresh
utils.VtGate().start()
# insert and check data on shard 0
self._insert_data('0', 100, 10)
self._check_data('0', 100, 10)
# insert and check data on shard 1
self._insert_data('1', 200, 10)
self._check_data('1', 200, 10)
# create a second table on all shards
sql = '''create table data2(
id bigint auto_increment,
name varchar(64),
primary key (id)
) Engine=InnoDB'''
utils.run_vtctl(['ApplySchema', '-sql=' + sql, 'test_keyspace'],
auto_log=True)
# reload schema everywhere so the QueryService knows about the tables
for t in all_tablets:
utils.run_vtctl(['ReloadSchema', t.tablet_alias], auto_log=True)
# insert and read data on all shards
self._insert_data('0', 300, 10, table='data2')
self._insert_data('1', 400, 10, table='data2')
self._check_data('0', 300, 10, table='data2')
self._check_data('1', 400, 10, table='data2')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
self._check_shards_count_in_srv_keyspace(2)
# Now test SplitQuery API works (used in MapReduce usually, but bringing
# up a full MR-capable cluster is too much for this test environment)
sql = 'select id, name from data'
s = utils.vtgate.split_query(sql, 'test_keyspace', 4)
self.assertEqual(len(s), 4)
shard0count = 0
shard1count = 0
for q in s:
if q['shard_part']['shards'][0] == '0':
shard0count += 1
if q['shard_part']['shards'][0] == '1':
shard1count += 1
self.assertEqual(shard0count, 2)
self.assertEqual(shard1count, 2)
# run the queries, aggregate the results, make sure we have all rows
rows = {}
for q in s:
bindvars = {}
for name, value in q['query']['bind_variables'].iteritems():
# vtctl encodes bytes as base64.
bindvars[name] = int(base64.standard_b64decode(value['value']))
qr = utils.vtgate.execute_shards(
q['query']['sql'],
'test_keyspace', ','.join(q['shard_part']['shards']),
tablet_type='master', bindvars=bindvars)
for r in qr['rows']:
rows[int(r[0])] = r[1]
self.assertEqual(len(rows), 20)
expected = {}
for i in xrange(10):
expected[100 + i] = 'row %d' % (100 + i)
expected[200 + i] = 'row %d' % (200 + i)
self.assertEqual(rows, expected)
self._test_vtclient_execute_shards_fallback()
def test_resharding(self):
# create the keyspace with just one shard
shard_master.init_tablet('replica',
keyspace='test_keyspace',
shard='0',
tablet_index=0)
shard_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='0',
tablet_index=1)
shard_rdonly1.init_tablet('rdonly',
keyspace='test_keyspace',
shard='0',
tablet_index=2)
for t in [shard_master, shard_replica, shard_rdonly1]:
t.create_db('vt_test_keyspace')
# replica is not started, InitShardMaster should timeout
shard_master.start_vttablet(wait_for_state=None,
binlog_use_v3_resharding_mode=False)
shard_rdonly1.start_vttablet(wait_for_state=None,
binlog_use_v3_resharding_mode=False)
for t in [shard_master, shard_rdonly1]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work - expect fail
# because replica tablet is not up
_, stderr = utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
shard_master.tablet_alias
],
auto_log=True,
expect_fail=True)
self.assertIn('tablet test_nj-0000062345 ResetReplication failed',
stderr)
# start replica
shard_replica.start_vttablet(wait_for_state=None,
binlog_use_v3_resharding_mode=False)
shard_replica.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/0',
shard_master.tablet_alias
],
auto_log=True)
utils.wait_for_tablet_type(shard_replica.tablet_alias, 'replica')
utils.wait_for_tablet_type(shard_rdonly1.tablet_alias, 'rdonly')
for t in [shard_master, shard_replica, shard_rdonly1]:
t.wait_for_vttablet_state('SERVING')
# create the tables and add startup values
self._create_schema()
self._insert_startup_values()
# reload schema on all tablets so we can query them
for t in [shard_master, shard_replica, shard_rdonly1]:
utils.run_vtctl(['ReloadSchema', t.tablet_alias], auto_log=True)
# We must start vtgate after tablets are up, or else wait until 1min refresh
# (that is the tablet_refresh_interval parameter for discovery gateway)
# we want cache_ttl at zero so we re-read the topology for every test query.
utils.VtGate().start(
cache_ttl='0',
tablets=[shard_master, shard_replica, shard_rdonly1])
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1)
# check the Map Reduce API works correctly, should use ExecuteShards,
# as we're not sharded yet.
# we have 3 values in the database, asking for 4 splits will get us
# a single query.
sql = 'select id, msg from resharding1'
s = utils.vtgate.split_query(sql, 'test_keyspace', 4)
self.assertEqual(len(s), 1)
self.assertEqual(s[0]['shard_part']['shards'][0], '0')
# change the schema, backfill keyspace_id, and change schema again
self._add_sharding_key_to_schema()
self._backfill_keyspace_id(shard_master)
self._mark_sharding_key_not_null()
# now we can be a sharded keyspace (and propagate to SrvKeyspace)
utils.run_vtctl([
'SetKeyspaceShardingInfo', 'test_keyspace', 'custom_ksid_col',
base_sharding.keyspace_id_type
])
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# run a health check on source replica so it responds to discovery
utils.run_vtctl(['RunHealthCheck', shard_replica.tablet_alias])
# create the split shards
shard_0_master.init_tablet('replica',
keyspace='test_keyspace',
shard='-80',
tablet_index=0)
shard_0_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='-80',
tablet_index=1)
shard_0_rdonly1.init_tablet('rdonly',
keyspace='test_keyspace',
shard='-80',
tablet_index=2)
shard_1_master.init_tablet('replica',
keyspace='test_keyspace',
shard='80-',
tablet_index=0)
shard_1_replica.init_tablet('replica',
keyspace='test_keyspace',
shard='80-',
tablet_index=1)
shard_1_rdonly1.init_tablet('rdonly',
keyspace='test_keyspace',
shard='80-',
tablet_index=2)
for t in [
shard_0_master, shard_0_replica, shard_0_rdonly1,
shard_1_master, shard_1_replica, shard_1_rdonly1
]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None,
binlog_use_v3_resharding_mode=False)
for t in [
shard_0_master, shard_0_replica, shard_0_rdonly1,
shard_1_master, shard_1_replica, shard_1_rdonly1
]:
t.wait_for_vttablet_state('NOT_SERVING')
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/-80',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'InitShardMaster', '-force', 'test_keyspace/80-',
shard_1_master.tablet_alias
],
auto_log=True)
for t in [shard_0_replica, shard_1_replica]:
utils.wait_for_tablet_type(t.tablet_alias, 'replica')
for t in [shard_0_rdonly1, shard_1_rdonly1]:
utils.wait_for_tablet_type(t.tablet_alias, 'rdonly')
sharded_tablets = [
shard_0_master, shard_0_replica, shard_0_rdonly1, shard_1_master,
shard_1_replica, shard_1_rdonly1
]
for t in sharded_tablets:
t.wait_for_vttablet_state('SERVING')
# must restart vtgate after tablets are up, or else wait until 1min refresh
# we want cache_ttl at zero so we re-read the topology for every test query.
utils.vtgate.kill()
utils.vtgate = None
utils.VtGate().start(cache_ttl='0',
tablets=[
shard_master, shard_replica, shard_rdonly1,
shard_0_master, shard_0_replica,
shard_0_rdonly1, shard_1_master,
shard_1_replica, shard_1_rdonly1
])
var = None
# Wait for the endpoints, either local or remote.
utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1, var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1, var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1, var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.master', 1, var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.replica',
1,
var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.-80.rdonly', 1, var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.master', 1, var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.replica',
1,
var=var)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.rdonly', 1, var=var)
# check the Map Reduce API works correctly, should use ExecuteKeyRanges now,
# as we are sharded (with just one shard).
# again, we have 3 values in the database, asking for 4 splits will get us
# a single query.
sql = 'select id, msg from resharding1'
s = utils.vtgate.split_query(sql, 'test_keyspace', 4)
self.assertEqual(len(s), 1)
self.assertEqual(s[0]['key_range_part']['keyspace'], 'test_keyspace')
# There must be one empty KeyRange which represents the full keyspace.
self.assertEqual(len(s[0]['key_range_part']['key_ranges']), 1)
self.assertEqual(s[0]['key_range_part']['key_ranges'][0], {})
utils.check_srv_keyspace(
'test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -\n'
'Partitions(replica): -\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# we need to create the schema, and the worker will do data copying
for keyspace_shard in ('test_keyspace/-80', 'test_keyspace/80-'):
utils.run_vtctl([
'CopySchemaShard', '--exclude_tables', 'unrelated',
shard_rdonly1.tablet_alias, keyspace_shard
],
auto_log=True)
utils.run_vtctl(['RunHealthCheck', shard_rdonly1.tablet_alias])
# Run vtworker as daemon for the following SplitClone commands.
worker_proc, worker_port, worker_rpc_port = utils.run_vtworker_bg(
[
'--cell', 'test_nj', '--command_display_interval', '10ms',
'--use_v3_resharding_mode=false'
],
auto_log=True)
# Initial clone (online).
workerclient_proc = utils.run_vtworker_client_bg([
'SplitClone', '--offline=false', '--exclude_tables', 'unrelated',
'--chunk_count', '10', '--min_rows_per_chunk', '1',
'--min_healthy_rdonly_tablets', '1', 'test_keyspace/0'
], worker_rpc_port)
utils.wait_procs([workerclient_proc])
self.verify_reconciliation_counters(worker_port, 'Online',
'resharding1', 3, 0, 0, 0)
# Reset vtworker such that we can run the next command.
workerclient_proc = utils.run_vtworker_client_bg(['Reset'],
worker_rpc_port)
utils.wait_procs([workerclient_proc])
# Modify the destination shard. SplitClone will revert the changes.
# Delete row 1 (provokes an insert).
shard_0_master.mquery('vt_test_keyspace',
'delete from resharding1 where id=1',
write=True)
# Delete row 2 (provokes an insert).
shard_1_master.mquery('vt_test_keyspace',
'delete from resharding1 where id=2',
write=True)
# Update row 3 (provokes an update).
shard_1_master.mquery(
'vt_test_keyspace',
"update resharding1 set msg='msg-not-3' where id=3",
write=True)
# Insert row 4 (provokes a delete).
self._insert_value(shard_1_master, 'resharding1', 4, 'msg4',
0xD000000000000000)
workerclient_proc = utils.run_vtworker_client_bg([
'SplitClone', '--exclude_tables', 'unrelated', '--chunk_count',
'10', '--min_rows_per_chunk', '1', '--min_healthy_rdonly_tablets',
'1', 'test_keyspace/0'
], worker_rpc_port)
utils.wait_procs([workerclient_proc])
self.verify_reconciliation_counters(worker_port, 'Online',
'resharding1', 2, 1, 1, 0)
self.verify_reconciliation_counters(worker_port, 'Offline',
'resharding1', 0, 0, 0, 3)
# Terminate worker daemon because it is no longer needed.
utils.kill_sub_process(worker_proc, soft=True)
# check the startup values are in the right place
self._check_startup_values()
# check the schema too
utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
auto_log=True)
# check the binlog players are running
logging.debug('Waiting for binlog players to start on new masters...')
self.check_destination_master(shard_0_master, ['test_keyspace/0'])
self.check_destination_master(shard_1_master, ['test_keyspace/0'])
# check that binlog server exported the stats vars
self.check_binlog_server_vars(shard_replica, horizontal=True)
# testing filtered replication: insert a bunch of data on shard 1,
# check we get most of it after a few seconds, wait for binlog server
# timeout, check we get all of it.
logging.debug('Inserting lots of data on source shard')
self._insert_lots(1000)
logging.debug('Checking 80 percent of data is sent quickly')
v = self._check_lots_timeout(1000, 80, 5)
if v != 100:
logging.debug('Checking all data goes through eventually')
self._check_lots_timeout(1000, 100, 20)
logging.debug('Checking no data was sent the wrong way')
self._check_lots_not_present(1000)
self.check_binlog_player_vars(shard_0_master, ['test_keyspace/0'],
seconds_behind_master_max=30)
self.check_binlog_player_vars(shard_1_master, ['test_keyspace/0'],
seconds_behind_master_max=30)
self.check_binlog_server_vars(shard_replica,
horizontal=True,
min_statements=1000,
min_transactions=1000)
# use vtworker to compare the data
for t in [shard_0_rdonly1, shard_1_rdonly1]:
utils.run_vtctl(['RunHealthCheck', t.tablet_alias])
if base_sharding.use_multi_split_diff:
logging.debug('Running vtworker MultiSplitDiff for 0')
utils.run_vtworker([
'-cell', 'test_nj', '--use_v3_resharding_mode=false',
'MultiSplitDiff', '--min_healthy_rdonly_tablets', '1',
'test_keyspace/0'
],
auto_log=True)
else:
logging.debug('Running vtworker SplitDiff for -80')
utils.run_vtworker([
'-cell', 'test_nj', '--use_v3_resharding_mode=false',
'SplitDiff', '--min_healthy_rdonly_tablets', '1',
'test_keyspace/-80'
],
auto_log=True)
logging.debug('Running vtworker SplitDiff for 80-')
utils.run_vtworker([
'-cell', 'test_nj', '--use_v3_resharding_mode=false',
'SplitDiff', '--min_healthy_rdonly_tablets', '1',
'test_keyspace/80-'
],
auto_log=True)
utils.pause('Good time to test vtworker for diffs')
# get status for the destination master tablet, make sure we have it all
self.check_running_binlog_player(shard_0_master, 2000, 2000)
self.check_running_binlog_player(shard_1_master, 6000, 2000)
# check we can't migrate the master just yet
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'master'],
expect_fail=True)
# now serve rdonly from the split shards
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'rdonly'],
auto_log=True)
utils.check_srv_keyspace(
'test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# make sure rdonly tablets are back to serving before hitting vtgate.
for t in [shard_0_rdonly1, shard_1_rdonly1]:
t.wait_for_vttablet_state('SERVING')
utils.vtgate.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
utils.vtgate.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
# check the Map Reduce API works correctly, should use ExecuteKeyRanges
# on both destination shards now.
# we ask for 2 splits to only have one per shard
sql = 'select id, msg from resharding1'
timeout = 10.0
while True:
try:
s = utils.vtgate.split_query(sql, 'test_keyspace', 2)
break
except Exception: # pylint: disable=broad-except
timeout = utils.wait_step(
'vtgate executes split_query properly', timeout)
self.assertEqual(len(s), 2)
self.assertEqual(s[0]['key_range_part']['keyspace'], 'test_keyspace')
self.assertEqual(s[1]['key_range_part']['keyspace'], 'test_keyspace')
self.assertEqual(len(s[0]['key_range_part']['key_ranges']), 1)
self.assertEqual(len(s[1]['key_range_part']['key_ranges']), 1)
# then serve replica from the split shards
source_tablet = shard_replica
destination_tablets = [shard_0_replica, shard_1_replica]
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'replica'],
auto_log=True)
utils.check_srv_keyspace(
'test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# move replica back and forth
utils.run_vtctl(
['MigrateServedTypes', '-reverse', 'test_keyspace/0', 'replica'],
auto_log=True)
# After a backwards migration, queryservice should be enabled on
# source and disabled on destinations
utils.check_tablet_query_service(self, source_tablet, True, False)
utils.check_tablet_query_services(self, destination_tablets, False,
True)
utils.check_srv_keyspace(
'test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'replica'],
auto_log=True)
# After a forwards migration, queryservice should be disabled on
# source and enabled on destinations
utils.check_tablet_query_service(self, source_tablet, False, True)
utils.check_tablet_query_services(self, destination_tablets, True,
False)
utils.check_srv_keyspace(
'test_nj',
'test_keyspace', 'Partitions(master): -\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# then serve master from the split shards
utils.run_vtctl(['MigrateServedTypes', 'test_keyspace/0', 'master'],
auto_log=True)
utils.check_srv_keyspace(
'test_nj',
'test_keyspace', 'Partitions(master): -80 80-\n'
'Partitions(rdonly): -80 80-\n'
'Partitions(replica): -80 80-\n',
keyspace_id_type=base_sharding.keyspace_id_type,
sharding_column_name='custom_ksid_col')
# check the binlog players are gone now
self.check_no_binlog_player(shard_0_master)
self.check_no_binlog_player(shard_1_master)
# make sure we can't delete a shard with tablets
utils.run_vtctl(['DeleteShard', 'test_keyspace/0'], expect_fail=True)
# remove the original tablets in the original shard
tablet.kill_tablets([shard_master, shard_replica, shard_rdonly1])
for t in [shard_replica, shard_rdonly1]:
utils.run_vtctl(['DeleteTablet', t.tablet_alias], auto_log=True)
utils.run_vtctl(
['DeleteTablet', '-allow_master', shard_master.tablet_alias],
auto_log=True)
# rebuild the serving graph, all mentions of the old shards should be gone
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# delete the original shard
utils.run_vtctl(['DeleteShard', 'test_keyspace/0'], auto_log=True)
# kill everything else
tablet.kill_tablets([
shard_0_master, shard_0_replica, shard_0_rdonly1, shard_1_master,
shard_1_replica, shard_1_rdonly1
])
class TestSecure(unittest.TestCase):
"""This test makes sure that we can use full TLS security within Vitess.
"""
def test_secure(self):
with open(table_acl_config, 'w') as fd:
fd.write("""{
"table_groups": [
{
"table_names_or_prefixes": ["vt_insert_test"],
"readers": ["vtgate client 1"],
"writers": ["vtgate client 1"],
"admins": ["vtgate client 1"]
}
]
}
""")
# start the tablets
shard_0_master.start_vttablet(
wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config,
extra_args=server_extra_args('vttablet-server-instance',
'vttablet-client'))
shard_0_slave.start_vttablet(
wait_for_state='NOT_SERVING',
table_acl_config=table_acl_config,
extra_args=server_extra_args('vttablet-server-instance',
'vttablet-client'))
# setup replication
utils.run_vtctl(tmclient_extra_args('vttablet-client-1') + [
'InitShardMaster', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias], auto_log=True)
utils.run_vtctl(tmclient_extra_args('vttablet-client-1') + [
'ApplySchema', '-sql', create_vt_insert_test,
'test_keyspace'])
for t in [shard_0_master, shard_0_slave]:
utils.run_vtctl(tmclient_extra_args('vttablet-client-1') + [
'RunHealthCheck', t.tablet_alias])
# start vtgate
utils.VtGate().start(extra_args=tabletconn_extra_args('vttablet-client-1')+
server_extra_args('vtgate-server-instance',
'vtgate-client'))
# 'vtgate client 1' is authorized to access vt_insert_test
protocol, addr = utils.vtgate.rpc_endpoint(python=True)
conn = vtgate_client.connect(protocol, addr, 30.0,
**python_client_kwargs('vtgate-client-1',
'vtgate-server'))
cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace',
shards=['0'])
cursor.execute('select * from vt_insert_test', {})
conn.close()
# 'vtgate client 2' is not authorized to access vt_insert_test
conn = vtgate_client.connect(protocol, addr, 30.0,
**python_client_kwargs('vtgate-client-2',
'vtgate-server'))
try:
cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace',
shards=['0'])
cursor.execute('select * from vt_insert_test', {})
self.fail('Execute went through')
except dbexceptions.DatabaseError, e:
s = str(e)
self.assertIn('table acl error', s)
self.assertIn('cannot run PASS_SELECT on table', s)
conn.close()
# now restart vtgate in the mode where we don't use SSL
# for client connections, but we copy effective caller id
# into immediate caller id.
utils.vtgate.kill()
utils.VtGate().start(extra_args=tabletconn_extra_args('vttablet-client-1')+
['-grpc_use_effective_callerid'])
protocol, addr = utils.vtgate.rpc_endpoint(python=True)
conn = vtgate_client.connect(protocol, addr, 30.0)
cursor = conn.cursor(tablet_type='master', keyspace='test_keyspace',
shards=['0'])
# not passing any immediate caller id should fail as using
# the unsecure user "unsecure_grpc_client"
cursor.set_effective_caller_id(None)
try:
cursor.execute('select * from vt_insert_test', {})
self.fail('Execute went through')
except dbexceptions.DatabaseError, e:
s = str(e)
self.assertIn('table acl error', s)
self.assertIn('cannot run PASS_SELECT on table', s)
self.assertIn('unsecure_grpc_client', s)