def _test_reparent_from_outside_check(self, brutal):
# make sure the replication graph is fine
shard_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0')
logging.debug('shard_files: %s' % " ".join(shard_files))
if shard_files != ['action', 'actionlog', 'test_nj-0000062044']:
raise utils.TestError('unexpected zk content: %s' % " ".join(shard_files))
slave_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044')
logging.debug('slave_files: %s' % " ".join(slave_files))
expected_slave_files = ['test_nj-0000041983', 'test_nj-0000062344']
if brutal:
expected_slave_files = ['test_nj-0000041983']
if slave_files != expected_slave_files:
raise utils.TestError('unexpected zk content: %s instead of expected %s' %
("|".join(slave_files),
"|".join(expected_slave_files)))
def _test_reparent_from_outside_check(self, brutal):
# make sure the replication graph is fine
shard_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0')
logging.debug('shard_files: %s' % " ".join(shard_files))
if shard_files != ['action', 'actionlog', 'test_nj-0000062044']:
raise utils.TestError('unexpected zk content: %s' % " ".join(shard_files))
slave_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044')
logging.debug('slave_files: %s' % " ".join(slave_files))
expected_slave_files = ['test_nj-0000041983', 'test_nj-0000062344']
if brutal:
expected_slave_files = ['test_nj-0000041983']
if slave_files != expected_slave_files:
raise utils.TestError('unexpected zk content: %s instead of expected %s' %
("|".join(slave_files),
"|".join(expected_slave_files_files)))
def run_test_reparent_from_outside():
utils.zk_wipe()
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True)
# Reparent as a starting point
utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias)
# now manually reparent 1 out of 2 tablets
# 62044 will be the new master
# 31981 won't be re-parented, so it w2ill be busted
tablet_62044.mquery('', [
"RESET MASTER",
"STOP SLAVE",
"RESET SLAVE",
"CHANGE MASTER TO MASTER_HOST = ''",
])
new_pos = tablet_62044.mquery('', 'show master status')
utils.debug("New master position: %s" % str(new_pos))
# 62344 will now be a slave of 62044
tablet_62344.mquery('', [
"RESET MASTER",
"RESET SLAVE",
"change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# 41983 will be a slave of 62044
tablet_41983.mquery('', [
'stop slave',
"change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# update zk with the new graph
utils.run_vtctl('ShardExternallyReparented -scrap-stragglers test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True)
# make sure the replication graph is fine
shard_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0')
utils.debug('shard_files: %s' % " ".join(shard_files))
if shard_files != ['action', 'actionlog', 'test_nj-0000062044']:
raise utils.TestError('unexpected zk content: %s' % " ".join(shard_files))
slave_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044')
utils.debug('slave_files: %s' % " ".join(slave_files))
if slave_files != ['test_nj-0000041983', 'test_nj-0000062344']:
raise utils.TestError('unexpected zk content: %s' % " ".join(slave_files))
tablet_31981.kill_vttablet()
tablet_62344.kill_vttablet()
tablet_62044.kill_vttablet()
tablet_41983.kill_vttablet()
def test_complex_schema(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
shard_0_master.init_tablet( 'master', 'test_keyspace', '0')
shard_0_replica1.init_tablet('replica', 'test_keyspace', '0')
shard_0_replica2.init_tablet('replica', 'test_keyspace', '0')
shard_0_rdonly.init_tablet( 'rdonly', 'test_keyspace', '0')
shard_0_backup.init_tablet( 'backup', 'test_keyspace', '0')
shard_1_master.init_tablet( 'master', 'test_keyspace', '1')
shard_1_replica1.init_tablet('replica', 'test_keyspace', '1')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets
for t in [shard_0_master, shard_0_replica1, shard_0_replica2,
shard_0_rdonly, shard_0_backup, shard_1_master, shard_1_replica1]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
# wait for the tablets to start
shard_0_master.wait_for_vttablet_state('SERVING')
shard_0_replica1.wait_for_vttablet_state('SERVING')
shard_0_replica2.wait_for_vttablet_state('SERVING')
shard_0_rdonly.wait_for_vttablet_state('SERVING')
shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
shard_1_master.wait_for_vttablet_state('SERVING')
shard_1_replica1.wait_for_vttablet_state('SERVING')
# make sure all replication is good
for t in [shard_0_master, shard_0_replica1, shard_0_replica2,
shard_0_rdonly, shard_0_backup, shard_1_master, shard_1_replica1]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/1 ' + shard_1_master.tablet_alias, auto_log=True)
utils.run_vtctl('ValidateKeyspace -ping-tablets test_keyspace')
# check after all tablets are here and replication is fixed
utils.validate_topology(ping_tablets=True)
# shard 0: apply the schema using a complex schema upgrade, no
# reparenting yet
utils.run_vtctl(['ApplySchemaShard',
'-sql='+create_vt_select_test[0],
'test_keyspace/0'],
auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
self._check_tables(shard_0_master, 0)
self._check_tables(shard_0_replica1, 1)
self._check_tables(shard_0_replica2, 1)
self._check_tables(shard_0_rdonly, 1)
self._check_tables(shard_0_backup, 1)
self._check_tables(shard_1_master, 0)
self._check_tables(shard_1_replica1, 0)
# shard 0: apply schema change to just master directly
# (to test its state is not changed)
utils.run_vtctl(['ApplySchema',
'-stop-replication',
'-sql='+create_vt_select_test[0],
shard_0_master.tablet_alias],
auto_log=True)
self._check_tables(shard_0_master, 1)
# shard 0: apply new schema change, with reparenting
utils.run_vtctl(['ApplySchemaShard',
'-new-parent='+shard_0_replica1.tablet_alias,
'-sql='+create_vt_select_test[1],
'test_keyspace/0'],
auto_log=True)
self._check_tables(shard_0_master, 1)
self._check_tables(shard_0_replica1, 2)
self._check_tables(shard_0_replica2, 2)
self._check_tables(shard_0_rdonly, 2)
self._check_tables(shard_0_backup, 2)
# verify GetSchema --tables works
out, err = utils.run_vtctl('GetSchema --tables=vt_select_test0 ' +
shard_0_replica1.tablet_alias,
log_level='INFO',
trap_output=True)
if not "vt_select_test0" in err or "vt_select_test1" in err:
self.fail('Unexpected GetSchema --tables=vt_select_test0 output: %s' % err)
# keyspace: try to apply a keyspace-wide schema change, should fail
# as the preflight would be different in both shards
out, err = utils.run_vtctl(['ApplySchemaKeyspace',
'-sql='+create_vt_select_test[2],
'test_keyspace'],
trap_output=True,
log_level='INFO',
raise_on_error=False)
if err.find('ApplySchemaKeyspace Shard 1 has inconsistent schema') == -1:
self.fail('Unexpected ApplySchemaKeyspace output: %s' % err)
if environment.topo_server_implementation == 'zookeeper':
utils.run_vtctl('PurgeActions /zk/global/vt/keyspaces/test_keyspace/action')
# shard 1: catch it up with simple updates
utils.run_vtctl(['ApplySchemaShard',
'-simple',
'-sql='+create_vt_select_test[0],
'test_keyspace/1'],
auto_log=True)
utils.run_vtctl(['ApplySchemaShard',
'-simple',
'-sql='+create_vt_select_test[1],
'test_keyspace/1'],
auto_log=True)
self._check_tables(shard_1_master, 2)
self._check_tables(shard_1_replica1, 2)
# keyspace: apply a keyspace-wide simple schema change, should work now
utils.run_vtctl(['ApplySchemaKeyspace',
'-simple',
'-sql='+create_vt_select_test[2],
'test_keyspace'],
auto_log=True)
# check all expected hosts have the change
self._check_tables(shard_0_master, 1) # was stuck a long time ago as scrap
self._check_tables(shard_0_replica1, 3) # current master
self._check_tables(shard_0_replica2, 3)
self._check_tables(shard_0_rdonly, 3)
self._check_tables(shard_0_backup, 3)
self._check_tables(shard_1_master, 3) # current master
self._check_tables(shard_1_replica1, 3)
# keyspace: apply a keyspace-wide complex schema change, should work too
utils.run_vtctl(['ApplySchemaKeyspace',
'-sql='+create_vt_select_test[3],
'test_keyspace'],
auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
# - backup won't have it as IsReplicatingType is false
self._check_tables(shard_0_master, 1) # was stuck a long time ago as scrap
self._check_tables(shard_0_replica1, 3) # current master
self._check_tables(shard_0_replica2, 4)
self._check_tables(shard_0_rdonly, 4)
self._check_tables(shard_0_backup, 4)
self._check_tables(shard_1_master, 3) # current master
self._check_tables(shard_1_replica1, 4)
# now test action log pruning
if environment.topo_server_implementation == 'zookeeper':
oldLines = utils.zk_ls(shard_0_replica1.zk_tablet_path+'/actionlog')
oldCount = len(oldLines)
logging.debug("I have %u actionlog before", oldCount)
if oldCount <= 5:
self.fail('Not enough actionlog before: %u' % oldCount)
utils.run_vtctl('PruneActionLogs -keep-count=5 /zk/*/vt/tablets/*/actionlog', auto_log=True)
newLines = utils.zk_ls(shard_0_replica1.zk_tablet_path+'/actionlog')
newCount = len(newLines)
logging.debug("I have %u actionlog after", newCount)
self.assertEqual(newCount, 5, 'Unexpected actionlog count after: %u' % newCount)
if oldLines[-5:] != newLines:
self.fail('Unexpected actionlog values:\n%s\n%s' %
(' '.join(oldLines[-5:]), ' '.join(newLines)))
utils.pause("Look at schema now!")
tablet.kill_tablets([shard_0_master, shard_0_replica1, shard_0_replica2,
shard_0_rdonly, shard_0_backup, shard_1_master,
shard_1_replica1])
def test_complex_schema(self):
utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
shard_0_master.init_tablet('master', 'test_keyspace', '0')
shard_0_replica1.init_tablet('replica', 'test_keyspace', '0')
shard_0_replica2.init_tablet('replica', 'test_keyspace', '0')
shard_0_rdonly.init_tablet('rdonly', 'test_keyspace', '0')
shard_0_backup.init_tablet('backup', 'test_keyspace', '0')
shard_1_master.init_tablet('master', 'test_keyspace', '1')
shard_1_replica1.init_tablet('replica', 'test_keyspace', '1')
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases, start the tablets
for t in [
shard_0_master, shard_0_replica1, shard_0_replica2,
shard_0_rdonly, shard_0_backup, shard_1_master,
shard_1_replica1
]:
t.create_db('vt_test_keyspace')
t.start_vttablet(wait_for_state=None)
# wait for the tablets to start
shard_0_master.wait_for_vttablet_state('SERVING')
shard_0_replica1.wait_for_vttablet_state('SERVING')
shard_0_replica2.wait_for_vttablet_state('SERVING')
shard_0_rdonly.wait_for_vttablet_state('SERVING')
shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
shard_1_master.wait_for_vttablet_state('SERVING')
shard_1_replica1.wait_for_vttablet_state('SERVING')
# make sure all replication is good
for t in [
shard_0_master, shard_0_replica1, shard_0_replica2,
shard_0_rdonly, shard_0_backup, shard_1_master,
shard_1_replica1
]:
t.reset_replication()
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/0',
shard_0_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', 'test_keyspace/1',
shard_1_master.tablet_alias
],
auto_log=True)
utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', 'test_keyspace'])
# check after all tablets are here and replication is fixed
utils.validate_topology(ping_tablets=True)
# shard 0: apply the schema using a complex schema upgrade, no
# reparenting yet
utils.run_vtctl([
'ApplySchemaShard', '-sql=' + create_vt_select_test[0],
'test_keyspace/0'
],
auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
self._check_tables(shard_0_master, 0)
self._check_tables(shard_0_replica1, 1)
self._check_tables(shard_0_replica2, 1)
self._check_tables(shard_0_rdonly, 1)
self._check_tables(shard_0_backup, 1)
self._check_tables(shard_1_master, 0)
self._check_tables(shard_1_replica1, 0)
# shard 0: apply schema change to just master directly
# (to test its state is not changed)
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test[0], shard_0_master.tablet_alias
],
auto_log=True)
self._check_tables(shard_0_master, 1)
# shard 0: apply new schema change, with reparenting
utils.run_vtctl([
'ApplySchemaShard', '-new-parent=' + shard_0_replica1.tablet_alias,
'-sql=' + create_vt_select_test[1], 'test_keyspace/0'
],
auto_log=True)
self._check_tables(shard_0_master, 1)
self._check_tables(shard_0_replica1, 2)
self._check_tables(shard_0_replica2, 2)
self._check_tables(shard_0_rdonly, 2)
self._check_tables(shard_0_backup, 2)
# verify GetSchema --tables works
s = utils.run_vtctl_json([
'GetSchema', '--tables=vt_select_test0',
shard_0_replica1.tablet_alias
])
self.assertEqual(len(s['TableDefinitions']), 1)
self.assertEqual(s['TableDefinitions'][0]['Name'], 'vt_select_test0')
# keyspace: try to apply a keyspace-wide schema change, should fail
# as the preflight would be different in both shards
out, err = utils.run_vtctl([
'ApplySchemaKeyspace', '-sql=' + create_vt_select_test[2],
'test_keyspace'
],
trap_output=True,
log_level='INFO',
raise_on_error=False)
if err.find(
'ApplySchemaKeyspace Shard 1 has inconsistent schema') == -1:
self.fail('Unexpected ApplySchemaKeyspace output: %s' % err)
if environment.topo_server_implementation == 'zookeeper':
utils.run_vtctl([
'PurgeActions', '/zk/global/vt/keyspaces/test_keyspace/action'
])
# shard 1: catch it up with simple updates
utils.run_vtctl([
'ApplySchemaShard', '-simple', '-sql=' + create_vt_select_test[0],
'test_keyspace/1'
],
auto_log=True)
utils.run_vtctl([
'ApplySchemaShard', '-simple', '-sql=' + create_vt_select_test[1],
'test_keyspace/1'
],
auto_log=True)
self._check_tables(shard_1_master, 2)
self._check_tables(shard_1_replica1, 2)
# keyspace: apply a keyspace-wide simple schema change, should work now
utils.run_vtctl([
'ApplySchemaKeyspace', '-simple',
'-sql=' + create_vt_select_test[2], 'test_keyspace'
],
auto_log=True)
# check all expected hosts have the change
self._check_tables(shard_0_master,
1) # was stuck a long time ago as scrap
self._check_tables(shard_0_replica1, 3) # current master
self._check_tables(shard_0_replica2, 3)
self._check_tables(shard_0_rdonly, 3)
self._check_tables(shard_0_backup, 3)
self._check_tables(shard_1_master, 3) # current master
self._check_tables(shard_1_replica1, 3)
# keyspace: apply a keyspace-wide complex schema change, should work too
utils.run_vtctl([
'ApplySchemaKeyspace', '-sql=' + create_vt_select_test[3],
'test_keyspace'
],
auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
# - backup won't have it as IsReplicatingType is false
self._check_tables(shard_0_master,
1) # was stuck a long time ago as scrap
self._check_tables(shard_0_replica1, 3) # current master
self._check_tables(shard_0_replica2, 4)
self._check_tables(shard_0_rdonly, 4)
self._check_tables(shard_0_backup, 4)
self._check_tables(shard_1_master, 3) # current master
self._check_tables(shard_1_replica1, 4)
# now test action log pruning
if environment.topo_server_implementation == 'zookeeper':
oldLines = utils.zk_ls(shard_0_replica1.zk_tablet_path +
'/actionlog')
oldCount = len(oldLines)
logging.debug("I have %u actionlog before", oldCount)
if oldCount <= 5:
self.fail('Not enough actionlog before: %u' % oldCount)
utils.run_vtctl([
'PruneActionLogs', '-keep-count=5',
'/zk/*/vt/tablets/*/actionlog'
],
auto_log=True)
newLines = utils.zk_ls(shard_0_replica1.zk_tablet_path +
'/actionlog')
newCount = len(newLines)
logging.debug("I have %u actionlog after", newCount)
self.assertEqual(newCount, 5,
'Unexpected actionlog count after: %u' % newCount)
if oldLines[-5:] != newLines:
self.fail('Unexpected actionlog values:\n%s\n%s' %
(' '.join(oldLines[-5:]), ' '.join(newLines)))
utils.pause("Look at schema now!")
tablet.kill_tablets([
shard_0_master, shard_0_replica1, shard_0_replica2, shard_0_rdonly,
shard_0_backup, shard_1_master, shard_1_replica1
])
def _run_test_reparent_from_outside(brutal=False):
utils.zk_wipe()
utils.run_vtctl("CreateKeyspace test_keyspace")
# create the database so vttablets start, as they are serving
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.create_db("vt_test_keyspace")
# Start up a master mysql and vttablet
tablet_62344.init_tablet("master", "test_keyspace", "0", start=True)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet("replica", "test_keyspace", "0", start=True)
tablet_41983.init_tablet("replica", "test_keyspace", "0", start=True)
tablet_31981.init_tablet("replica", "test_keyspace", "0", start=True)
# Reparent as a starting point
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.run_vtctl("ReparentShard -force test_keyspace/0 %s" % tablet_62344.tablet_alias, auto_log=True)
# now manually reparent 1 out of 2 tablets
# 62044 will be the new master
# 31981 won't be re-parented, so it will be busted
tablet_62044.mquery("", ["RESET MASTER", "STOP SLAVE", "RESET SLAVE", "CHANGE MASTER TO MASTER_HOST = ''"])
new_pos = tablet_62044.mquery("", "show master status")
utils.debug("New master position: %s" % str(new_pos))
# 62344 will now be a slave of 62044
tablet_62344.mquery(
"",
[
"RESET MASTER",
"RESET SLAVE",
"change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u"
% (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
"start slave",
],
)
# 41983 will be a slave of 62044
tablet_41983.mquery(
"",
[
"stop slave",
"change master to master_port=%u, master_log_file='%s', master_log_pos=%u"
% (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
"start slave",
],
)
# in brutal mode, we scrap the old master first
if brutal:
tablet_62344.scrap(force=True)
# we have some automated tools that do this too, so it's good to simulate
utils.run(utils.vtroot + "/bin/zk rm -rf " + tablet_62344.zk_tablet_path)
# update zk with the new graph
utils.run_vtctl(
"ShardExternallyReparented -scrap-stragglers test_keyspace/0 %s" % tablet_62044.tablet_alias, auto_log=True
)
# make sure the replication graph is fine
shard_files = utils.zk_ls("/zk/global/vt/keyspaces/test_keyspace/shards/0")
utils.debug("shard_files: %s" % " ".join(shard_files))
if shard_files != ["action", "actionlog", "test_nj-0000062044"]:
raise utils.TestError("unexpected zk content: %s" % " ".join(shard_files))
slave_files = utils.zk_ls("/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044")
utils.debug("slave_files: %s" % " ".join(slave_files))
expected_slave_files = ["test_nj-0000041983", "test_nj-0000062344"]
if brutal:
expected_slave_files = ["test_nj-0000041983"]
if slave_files != expected_slave_files:
raise utils.TestError(
"unexpected zk content: %s instead of expected %s"
% ("|".join(slave_files), "|".join(expected_slave_files_files))
)
tablet_31981.kill_vttablet()
tablet_62344.kill_vttablet()
tablet_62044.kill_vttablet()
tablet_41983.kill_vttablet()
def _run_test_reparent_from_outside(brutal=False):
utils.zk_wipe()
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True)
# Reparent as a starting point
for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' %
tablet_62344.tablet_alias,
auto_log=True)
# now manually reparent 1 out of 2 tablets
# 62044 will be the new master
# 31981 won't be re-parented, so it will be busted
tablet_62044.mquery('', [
"RESET MASTER",
"STOP SLAVE",
"RESET SLAVE",
"CHANGE MASTER TO MASTER_HOST = ''",
])
new_pos = tablet_62044.mquery('', 'show master status')
utils.debug("New master position: %s" % str(new_pos))
# 62344 will now be a slave of 62044
tablet_62344.mquery('', [
"RESET MASTER", "RESET SLAVE",
"change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u"
% (utils.hostname, tablet_62044.mysql_port, new_pos[0][0],
new_pos[0][1]), 'start slave'
])
# 41983 will be a slave of 62044
tablet_41983.mquery('', [
'stop slave',
"change master to master_port=%u, master_log_file='%s', master_log_pos=%u"
%
(tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave'
])
# in brutal mode, we scrap the old master first
if brutal:
tablet_62344.scrap(force=True)
# we have some automated tools that do this too, so it's good to simulate
utils.run(utils.vtroot + '/bin/zk rm -rf ' +
tablet_62344.zk_tablet_path)
# update zk with the new graph
utils.run_vtctl(
'ShardExternallyReparented -scrap-stragglers test_keyspace/0 %s' %
tablet_62044.tablet_alias,
auto_log=True)
# make sure the replication graph is fine
shard_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0')
utils.debug('shard_files: %s' % " ".join(shard_files))
if shard_files != ['action', 'actionlog', 'test_nj-0000062044']:
raise utils.TestError('unexpected zk content: %s' %
" ".join(shard_files))
slave_files = utils.zk_ls(
'/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044')
utils.debug('slave_files: %s' % " ".join(slave_files))
expected_slave_files = ['test_nj-0000041983', 'test_nj-0000062344']
if brutal:
expected_slave_files = ['test_nj-0000041983']
if slave_files != expected_slave_files:
raise utils.TestError(
'unexpected zk content: %s instead of expected %s' %
("|".join(slave_files), "|".join(expected_slave_files_files)))
tablet_31981.kill_vttablet()
tablet_62344.kill_vttablet()
tablet_62044.kill_vttablet()
tablet_41983.kill_vttablet()
def run_test_complex_schema():
utils.run_vtctl("CreateKeyspace test_keyspace")
shard_0_master.init_tablet("master", "test_keyspace", "0")
shard_0_replica1.init_tablet("replica", "test_keyspace", "0")
shard_0_replica2.init_tablet("replica", "test_keyspace", "0")
shard_0_rdonly.init_tablet("rdonly", "test_keyspace", "0")
shard_0_backup.init_tablet("backup", "test_keyspace", "0")
shard_1_master.init_tablet("master", "test_keyspace", "1")
shard_1_replica1.init_tablet("replica", "test_keyspace", "1")
utils.run_vtctl("RebuildShardGraph test_keyspace/0", auto_log=True)
utils.run_vtctl("RebuildKeyspaceGraph test_keyspace", auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases
shard_0_master.create_db("vt_test_keyspace")
shard_0_replica1.create_db("vt_test_keyspace")
shard_0_replica2.create_db("vt_test_keyspace")
shard_0_rdonly.create_db("vt_test_keyspace")
shard_0_backup.create_db("vt_test_keyspace")
shard_1_master.create_db("vt_test_keyspace")
shard_1_replica1.create_db("vt_test_keyspace")
# start the tablets
shard_0_master.start_vttablet()
shard_0_replica1.start_vttablet()
shard_0_replica2.start_vttablet()
shard_0_rdonly.start_vttablet()
shard_0_backup.start_vttablet(wait_for_state="NOT_SERVING")
shard_1_master.start_vttablet()
shard_1_replica1.start_vttablet()
# make sure all replication is good
for t in [
shard_0_master,
shard_0_replica1,
shard_0_replica2,
shard_0_rdonly,
shard_0_backup,
shard_1_master,
shard_1_replica1,
]:
t.reset_replication()
utils.run_vtctl("ReparentShard -force test_keyspace/0 " + shard_0_master.tablet_alias, auto_log=True)
utils.run_vtctl("ReparentShard -force test_keyspace/1 " + shard_1_master.tablet_alias, auto_log=True)
utils.run_vtctl("ValidateKeyspace -ping-tablets test_keyspace")
# check after all tablets are here and replication is fixed
utils.validate_topology(ping_tablets=True)
# shard 0: apply the schema using a complex schema upgrade, no
# reparenting yet
utils.run_vtctl(["ApplySchemaShard", "-sql=" + create_vt_select_test[0], "test_keyspace/0"], auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
check_tables(shard_0_master, 0)
check_tables(shard_0_replica1, 1)
check_tables(shard_0_replica2, 1)
check_tables(shard_0_rdonly, 1)
check_tables(shard_0_backup, 1)
check_tables(shard_1_master, 0)
check_tables(shard_1_replica1, 0)
# shard 0: apply schema change to just master directly
# (to test its state is not changed)
utils.run_vtctl(
["ApplySchema", "-stop-replication", "-sql=" + create_vt_select_test[0], shard_0_master.tablet_alias],
auto_log=True,
)
check_tables(shard_0_master, 1)
# shard 0: apply new schema change, with reparenting
utils.run_vtctl(
[
"ApplySchemaShard",
"-new-parent=" + shard_0_replica1.tablet_alias,
"-sql=" + create_vt_select_test[1],
"test_keyspace/0",
],
auto_log=True,
)
check_tables(shard_0_master, 1)
check_tables(shard_0_replica1, 2)
check_tables(shard_0_replica2, 2)
check_tables(shard_0_rdonly, 2)
check_tables(shard_0_backup, 2)
# verify GetSchema --tables works
out, err = utils.run_vtctl(
"GetSchema --tables=vt_select_test0 " + shard_0_replica1.tablet_alias, log_level="INFO", trap_output=True
)
if not "vt_select_test0" in err or "vt_select_test1" in err:
raise utils.TestError("Unexpected GetSchema --tables=vt_select_test0 output: %s" % err)
# keyspace: try to apply a keyspace-wide schema change, should fail
# as the preflight would be different in both shards
out, err = utils.run_vtctl(
["ApplySchemaKeyspace", "-sql=" + create_vt_select_test[2], "test_keyspace"],
log_level="INFO",
trap_output=True,
raise_on_error=False,
)
if err.find("ApplySchemaKeyspace Shard 1 has inconsistent schema") == -1:
raise utils.TestError("Unexpected ApplySchemaKeyspace output: %s" % err)
utils.run_vtctl("PurgeActions /zk/global/vt/keyspaces/test_keyspace/action")
# shard 1: catch it up with simple updates
utils.run_vtctl(
["ApplySchemaShard", "-simple", "-sql=" + create_vt_select_test[0], "test_keyspace/1"], auto_log=True
)
utils.run_vtctl(
["ApplySchemaShard", "-simple", "-sql=" + create_vt_select_test[1], "test_keyspace/1"], auto_log=True
)
check_tables(shard_1_master, 2)
check_tables(shard_1_replica1, 2)
# keyspace: apply a keyspace-wide simple schema change, should work now
utils.run_vtctl(
["ApplySchemaKeyspace", "-simple", "-sql=" + create_vt_select_test[2], "test_keyspace"], auto_log=True
)
# check all expected hosts have the change
check_tables(shard_0_master, 1) # was stuck a long time ago as scrap
check_tables(shard_0_replica1, 3) # current master
check_tables(shard_0_replica2, 3)
check_tables(shard_0_rdonly, 3)
check_tables(shard_0_backup, 3)
check_tables(shard_1_master, 3) # current master
check_tables(shard_1_replica1, 3)
# keyspace: apply a keyspace-wide complex schema change, should work too
utils.run_vtctl(["ApplySchemaKeyspace", "-sql=" + create_vt_select_test[3], "test_keyspace"], auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
# - backup won't have it as IsReplicatingType is false
check_tables(shard_0_master, 1) # was stuck a long time ago as scrap
check_tables(shard_0_replica1, 3) # current master
check_tables(shard_0_replica2, 4)
check_tables(shard_0_rdonly, 4)
check_tables(shard_0_backup, 4)
check_tables(shard_1_master, 3) # current master
check_tables(shard_1_replica1, 4)
# now test action log pruning
oldLines = utils.zk_ls(shard_0_replica1.zk_tablet_path + "/actionlog")
oldCount = len(oldLines)
if utils.options.verbose:
print "I have %u actionlog before" % oldCount
if oldCount <= 5:
raise utils.TestError("Not enough actionlog before: %u" % oldCount)
utils.run_vtctl("PruneActionLogs -keep-count=5 /zk/*/vt/tablets/*/actionlog", auto_log=True)
newLines = utils.zk_ls(shard_0_replica1.zk_tablet_path + "/actionlog")
newCount = len(newLines)
if utils.options.verbose:
print "I have %u actionlog after" % newCount
if newCount != 5:
raise utils.TestError("Unexpected actionlog count after: %u" % newCount)
if oldLines[-5:] != newLines:
raise utils.TestError("Unexpected actionlog values:\n%s\n%s" % (" ".join(oldLines[-5:]), " ".join(newLines)))
utils.pause("Look at schema now!")
shard_0_master.kill_vttablet()
shard_0_replica1.kill_vttablet()
shard_0_replica2.kill_vttablet()
shard_0_rdonly.kill_vttablet()
shard_0_backup.kill_vttablet()
shard_1_master.kill_vttablet()
shard_1_replica1.kill_vttablet()
def run_test_complex_schema():
utils.run_vtctl('CreateKeyspace test_keyspace')
shard_0_master.init_tablet('master', 'test_keyspace', '0')
shard_0_replica1.init_tablet('replica', 'test_keyspace', '0')
shard_0_replica2.init_tablet('replica', 'test_keyspace', '0')
shard_0_rdonly.init_tablet('rdonly', 'test_keyspace', '0')
shard_0_backup.init_tablet('backup', 'test_keyspace', '0')
shard_1_master.init_tablet('master', 'test_keyspace', '1')
shard_1_replica1.init_tablet('replica', 'test_keyspace', '1')
utils.run_vtctl('RebuildShardGraph test_keyspace/0', auto_log=True)
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases
shard_0_master.create_db('vt_test_keyspace')
shard_0_replica1.create_db('vt_test_keyspace')
shard_0_replica2.create_db('vt_test_keyspace')
shard_0_rdonly.create_db('vt_test_keyspace')
shard_0_backup.create_db('vt_test_keyspace')
shard_1_master.create_db('vt_test_keyspace')
shard_1_replica1.create_db('vt_test_keyspace')
# start the tablets
shard_0_master.start_vttablet()
shard_0_replica1.start_vttablet()
shard_0_replica2.start_vttablet()
shard_0_rdonly.start_vttablet()
shard_0_backup.start_vttablet(wait_for_state="NOT_SERVING")
shard_1_master.start_vttablet()
shard_1_replica1.start_vttablet()
# make sure all replication is good
for t in [
shard_0_master, shard_0_replica1, shard_0_replica2, shard_0_rdonly,
shard_0_backup, shard_1_master, shard_1_replica1
]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
shard_0_master.tablet_alias,
auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/1 ' +
shard_1_master.tablet_alias,
auto_log=True)
utils.run_vtctl('ValidateKeyspace -ping-tablets test_keyspace')
# check after all tablets are here and replication is fixed
utils.validate_topology(ping_tablets=True)
# shard 0: apply the schema using a complex schema upgrade, no
# reparenting yet
utils.run_vtctl([
'ApplySchemaShard', '-sql=' + create_vt_select_test[0],
'test_keyspace/0'
],
auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
check_tables(shard_0_master, 0)
check_tables(shard_0_replica1, 1)
check_tables(shard_0_replica2, 1)
check_tables(shard_0_rdonly, 1)
check_tables(shard_0_backup, 1)
check_tables(shard_1_master, 0)
check_tables(shard_1_replica1, 0)
# shard 0: apply schema change to just master directly
# (to test its state is not changed)
utils.run_vtctl([
'ApplySchema', '-stop-replication', '-sql=' + create_vt_select_test[0],
shard_0_master.tablet_alias
],
auto_log=True)
check_tables(shard_0_master, 1)
# shard 0: apply new schema change, with reparenting
utils.run_vtctl([
'ApplySchemaShard', '-new-parent=' + shard_0_replica1.tablet_alias,
'-sql=' + create_vt_select_test[1], 'test_keyspace/0'
],
auto_log=True)
check_tables(shard_0_master, 1)
check_tables(shard_0_replica1, 2)
check_tables(shard_0_replica2, 2)
check_tables(shard_0_rdonly, 2)
check_tables(shard_0_backup, 2)
# verify GetSchema --tables works
out, err = utils.run_vtctl('GetSchema --tables=vt_select_test0 ' +
shard_0_replica1.tablet_alias,
trap_output=True)
if not "vt_select_test0" in err or "vt_select_test1" in err:
raise utils.TestError(
'Unexpected GetSchema --tables=vt_select_test0 output: %s' % err)
# keyspace: try to apply a keyspace-wide schema change, should fail
# as the preflight would be different in both shards
out, err = utils.run_vtctl([
'ApplySchemaKeyspace', '-sql=' + create_vt_select_test[2],
'test_keyspace'
],
trap_output=True,
raise_on_error=False)
if err.find('ApplySchemaKeyspace Shard 1 has inconsistent schema') == -1:
raise utils.TestError('Unexpected ApplySchemaKeyspace output: %s' %
err)
utils.run_vtctl(
'PurgeActions /zk/global/vt/keyspaces/test_keyspace/action')
# shard 1: catch it up with simple updates
utils.run_vtctl([
'ApplySchemaShard', '-simple', '-sql=' + create_vt_select_test[0],
'test_keyspace/1'
],
auto_log=True)
utils.run_vtctl([
'ApplySchemaShard', '-simple', '-sql=' + create_vt_select_test[1],
'test_keyspace/1'
],
auto_log=True)
check_tables(shard_1_master, 2)
check_tables(shard_1_replica1, 2)
# keyspace: apply a keyspace-wide simple schema change, should work now
utils.run_vtctl([
'ApplySchemaKeyspace', '-simple', '-sql=' + create_vt_select_test[2],
'test_keyspace'
],
auto_log=True)
# check all expected hosts have the change
check_tables(shard_0_master, 1) # was stuck a long time ago as scrap
check_tables(shard_0_replica1, 3) # current master
check_tables(shard_0_replica2, 3)
check_tables(shard_0_rdonly, 3)
check_tables(shard_0_backup, 3)
check_tables(shard_1_master, 3) # current master
check_tables(shard_1_replica1, 3)
# keyspace: apply a keyspace-wide complex schema change, should work too
utils.run_vtctl([
'ApplySchemaKeyspace', '-sql=' + create_vt_select_test[3],
'test_keyspace'
],
auto_log=True)
# check all expected hosts have the change:
# - master won't have it as it's a complex change
# - backup won't have it as IsReplicatingType is false
check_tables(shard_0_master, 1) # was stuck a long time ago as scrap
check_tables(shard_0_replica1, 3) # current master
check_tables(shard_0_replica2, 4)
check_tables(shard_0_rdonly, 4)
check_tables(shard_0_backup, 4)
check_tables(shard_1_master, 3) # current master
check_tables(shard_1_replica1, 4)
# now test action log pruning
oldLines = utils.zk_ls(shard_0_replica1.zk_tablet_path + '/actionlog')
oldCount = len(oldLines)
if utils.options.verbose:
print "I have %u actionlog before" % oldCount
if oldCount <= 5:
raise utils.TestError('Not enough actionlog before: %u' % oldCount)
utils.run_vtctl(
'PruneActionLogs -keep-count=5 /zk/*/vt/tablets/*/actionlog',
auto_log=True)
newLines = utils.zk_ls(shard_0_replica1.zk_tablet_path + '/actionlog')
newCount = len(newLines)
if utils.options.verbose:
print "I have %u actionlog after" % newCount
if newCount != 5:
raise utils.TestError('Unexpected actionlog count after: %u' %
newCount)
if oldLines[-5:] != newLines:
raise utils.TestError('Unexpected actionlog values:\n%s\n%s' %
(' '.join(oldLines[-5:]), ' '.join(newLines)))
utils.pause("Look at schema now!")
shard_0_master.kill_vttablet()
shard_0_replica1.kill_vttablet()
shard_0_replica2.kill_vttablet()
shard_0_rdonly.kill_vttablet()
shard_0_backup.kill_vttablet()
shard_1_master.kill_vttablet()
shard_1_replica1.kill_vttablet()
