def test_topo_read_threshold(self):
before_topo_rtt = get_topo_rtt()
# Check original state.
keyspace_obj = topology.get_keyspace("test_keyspace")
self.assertNotEqual(keyspace_obj, None, "test_keyspace should be not None")
self.assertEqual(
keyspace_obj.sharding_col_type,
keyrange_constants.KIT_UINT64,
"ShardingColumnType be %s" % keyrange_constants.KIT_UINT64,
)
# Change the keyspace object.
utils.run_vtctl(
["SetKeyspaceShardingInfo", "-force", "test_keyspace", "keyspace_id", keyrange_constants.KIT_BYTES]
)
utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)
# sleep throttle interval and check values again.
# the keyspace should have changed and also caused a rtt to topo server.
time.sleep(self.keyspace_fetch_throttle)
topology.refresh_keyspace(self.vtgate_client, "test_keyspace")
keyspace_obj = topology.get_keyspace("test_keyspace")
after_1st_clear = get_topo_rtt()
self.assertEqual(after_1st_clear - before_topo_rtt, 1, "One additional round-trips to topo server")
self.assertEqual(
keyspace_obj.sharding_col_type,
keyrange_constants.KIT_BYTES,
"ShardingColumnType be %s" % keyrange_constants.KIT_BYTES,
)
# Refresh without sleeping for throttle time shouldn't cause additional rtt.
topology.refresh_keyspace(self.vtgate_client, "test_keyspace")
keyspace_obj = topology.get_keyspace("test_keyspace")
after_2nd_clear = get_topo_rtt()
self.assertEqual(after_2nd_clear - after_1st_clear, 0, "No additional round-trips to topo server")
def test_topo_read_threshold(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
# Check original state.
keyspace_obj = topology.get_keyspace('test_keyspace')
self.assertNotEqual(keyspace_obj, None, "test_keyspace should be not None")
self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64, "ShardingColumnType be %s" % keyrange_constants.KIT_UINT64)
# Change the keyspace object.
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
'keyspace_id', keyrange_constants.KIT_BYTES])
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# sleep throttle interval and check values again.
# the keyspace should have changed and also caused a rtt to topo server.
time.sleep(self.keyspace_fetch_throttle)
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_1st_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_1st_clear - before_topo_rtt, 1, "One additional round-trips to topo server")
self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES, "ShardingColumnType be %s" % keyrange_constants.KIT_BYTES)
# Refresh without sleeping for throttle time shouldn't cause additional rtt.
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_2nd_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_2nd_clear - after_1st_clear, 0, "No additional round-trips to topo server")
def test_topo_read_threshold(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
# Check original state.
keyspace_obj = topology.get_keyspace('test_keyspace')
self.assertNotEqual(
keyspace_obj, None, 'test_keyspace should be not None')
self.assertEqual(
keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64,
'ShardingColumnType be %s' % keyrange_constants.KIT_UINT64)
# Change the keyspace object.
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
'keyspace_id', keyrange_constants.KIT_BYTES])
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# sleep throttle interval and check values again.
# the keyspace should have changed and also caused a rtt to topo server.
time.sleep(self.keyspace_fetch_throttle)
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_1st_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(
after_1st_clear - before_topo_rtt, 1,
'One additional round-trips to topo server')
self.assertEqual(
keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES,
'ShardingColumnType be %s' % keyrange_constants.KIT_BYTES)
# Refresh without sleeping for throttle time shouldn't cause
# additional rtt.
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_2nd_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(
after_2nd_clear - after_1st_clear, 0,
'No additional round-trips to topo server')
def resolve_topology(self):
topology.refresh_keyspace(self.topo_client, self.keyspace)
def test_vertical_split(self):
utils.run_vtctl(['CreateKeyspace', 'source_keyspace'])
utils.run_vtctl(['CreateKeyspace',
'--served-from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace',
'destination_keyspace'])
source_master.init_tablet('master', 'source_keyspace', '0')
source_replica.init_tablet('replica', 'source_keyspace', '0')
source_rdonly.init_tablet('rdonly', 'source_keyspace', '0')
# rebuild destination keyspace to make sure there is a serving
# graph entry, even though there is no tablet yet.
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
destination_master.init_tablet('master', 'destination_keyspace', '0')
destination_replica.init_tablet('replica', 'destination_keyspace', '0')
destination_rdonly.init_tablet('rdonly', 'destination_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
# create databases so vttablet can start behaving normally
for t in [source_master, source_replica, source_rdonly]:
t.create_db('vt_source_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [destination_master, destination_replica, destination_rdonly]:
t.start_vttablet(wait_for_state=None)
# wait for the tablets
for t in [source_master, source_replica, source_rdonly]:
t.wait_for_vttablet_state('SERVING')
for t in [destination_master, destination_replica, destination_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl(['ReparentShard', '-force', 'source_keyspace/0',
source_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', 'destination_keyspace/0',
destination_master.tablet_alias], auto_log=True)
# read all the keyspaces, this will populate the topology cache.
self._populate_topo_cache()
# create the schema on the source keyspace, add some values
self._create_source_schema()
moving1_first = self._insert_values('moving1', 100)
moving2_first = self._insert_values('moving2', 100)
staying1_first = self._insert_values('staying1', 100)
staying2_first = self._insert_values('staying2', 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving1',
moving1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving2',
moving2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying1',
staying1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying2',
staying2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'view1',
moving1_first, 100)
utils.pause("Before multisnapshot")
# take the snapshot for the split
utils.run_vtctl(['MultiSnapshot',
'--tables', 'moving.*,view1',
source_rdonly.tablet_alias], auto_log=True)
# perform the restore.
utils.run_vtctl(['ShardMultiRestore',
'--strategy' ,'populateBlpCheckpoint',
'--tables', 'moving.*',
'destination_keyspace/0', source_rdonly.tablet_alias],
auto_log=True)
topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace')
# check values are present
self._check_values(destination_master, 'vt_destination_keyspace', 'moving1',
moving1_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace', 'moving2',
moving2_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace', 'view1',
moving1_first, 100)
# check the binlog players is running
destination_master.wait_for_binlog_player_count(1)
# add values to source, make sure they're replicated
moving1_first_add1 = self._insert_values('moving1', 100)
staying1_first_add1 = self._insert_values('staying1', 100)
moving2_first_add1 = self._insert_values('moving2', 100)
self._check_values_timeout(destination_master, 'vt_destination_keyspace',
'moving1', moving1_first_add1, 100)
self._check_values_timeout(destination_master, 'vt_destination_keyspace',
'moving2', moving2_first_add1, 100)
# use the vtworker checker to compare the data
logging.debug("Running vtworker VerticalSplitDiff")
utils.run_vtworker(['-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', source_rdonly.tablet_alias, 'rdonly'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', destination_rdonly.tablet_alias,
'rdonly'], auto_log=True)
utils.pause("Good time to test vtworker for diffs")
# get status for destination master tablet, make sure we have it all
destination_master_status = destination_master.get_status()
self.assertIn('Binlog player state: Running', destination_master_status)
self.assertIn('moving.*', destination_master_status)
self.assertIn('<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status)
self.assertIn('</html>', destination_master_status)
# check we can't migrate the master just yet
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'],
expect_fail=True)
# now serve rdonly from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['rdonly'], ['master', 'replica'], ['moving1', 'moving2'])
# then serve replica from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*'])
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2'])
# move replica back and forth
utils.run_vtctl(['MigrateServedFrom', '-reverse', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*'])
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2'])
# then serve master from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'],
auto_log=True)
self._check_srv_keyspace('')
self._check_blacklisted_tables(source_master, ['moving.*'])
self._check_blacklisted_tables(source_replica, ['moving.*'])
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly', 'master'], [], ['moving1', 'moving2'])
# check 'vtctl SetBlacklistedTables' command works as expected
utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias,
'moving.*,view1'], auto_log=True)
self._check_blacklisted_tables(source_master, ['moving.*', 'view1'])
utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias],
auto_log=True)
self._check_blacklisted_tables(source_master, None)
# check the binlog player is gone now
destination_master.wait_for_binlog_player_count(0)
# optional method to check the stats are correct
self._check_stats()
# kill everything
tablet.kill_tablets([source_master, source_replica, source_rdonly,
destination_master, destination_replica,
destination_rdonly])
def refresh_keyspace(self, keyspace_name): # This is so that keyspace can be refreshed. time.sleep(self.throttle_sleep_interval) topology.refresh_keyspace(self.vtgate_client, keyspace_name)
def test_vertical_split(self):
utils.run_vtctl(['CreateKeyspace', 'source_keyspace'])
utils.run_vtctl(['CreateKeyspace',
'--served_from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace',
'destination_keyspace'])
source_master.init_tablet('master', 'source_keyspace', '0')
source_replica.init_tablet('replica', 'source_keyspace', '0')
source_rdonly1.init_tablet('rdonly', 'source_keyspace', '0')
source_rdonly2.init_tablet('rdonly', 'source_keyspace', '0')
# rebuild destination keyspace to make sure there is a serving
# graph entry, even though there is no tablet yet.
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
destination_master.init_tablet('master', 'destination_keyspace', '0')
destination_replica.init_tablet('replica', 'destination_keyspace', '0')
destination_rdonly1.init_tablet('rdonly', 'destination_keyspace', '0')
destination_rdonly2.init_tablet('rdonly', 'destination_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
# create databases so vttablet can start behaving normally
for t in [source_master, source_replica, source_rdonly1, source_rdonly2]:
t.create_db('vt_source_keyspace')
t.start_vttablet(wait_for_state=None)
destination_master.start_vttablet(wait_for_state=None,
target_tablet_type='replica')
for t in [destination_replica, destination_rdonly1, destination_rdonly2]:
t.start_vttablet(wait_for_state=None)
# wait for the tablets
for t in [source_master, source_replica, source_rdonly1, source_rdonly2]:
t.wait_for_vttablet_state('SERVING')
for t in [destination_master, destination_replica, destination_rdonly1,
destination_rdonly2]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl(['ReparentShard', '-force', 'source_keyspace/0',
source_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', 'destination_keyspace/0',
destination_master.tablet_alias], auto_log=True)
# read all the keyspaces, this will populate the topology cache.
self._populate_topo_cache()
# create the schema on the source keyspace, add some values
self._create_source_schema()
moving1_first = self._insert_values('moving1', 100)
moving2_first = self._insert_values('moving2', 100)
staying1_first = self._insert_values('staying1', 100)
staying2_first = self._insert_values('staying2', 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving1',
moving1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving2',
moving2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying1',
staying1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying2',
staying2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'view1',
moving1_first, 100)
# the worker will do everything. We test with source_reader_count=10
# (down from default=20) as connection pool is not big enough for 20.
# min_table_size_for_split is set to 1 as to force a split even on the
# small table we have.
utils.run_vtctl(['CopySchemaShard', '--tables', 'moving.*,view1',
source_rdonly1.tablet_alias, 'destination_keyspace/0'],
auto_log=True)
utils.run_vtworker(['--cell', 'test_nj',
'--command_display_interval', '10ms',
'VerticalSplitClone',
'--tables', 'moving.*,view1',
'--strategy=-populate_blp_checkpoint',
'--source_reader_count', '10',
'--min_table_size_for_split', '1',
'destination_keyspace/0'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', source_rdonly1.tablet_alias,
'rdonly'], auto_log=True)
utils.run_vtctl(['ChangeSlaveType', source_rdonly2.tablet_alias,
'rdonly'], auto_log=True)
topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace')
# check values are present
self._check_values(destination_master, 'vt_destination_keyspace', 'moving1',
moving1_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace', 'moving2',
moving2_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace', 'view1',
moving1_first, 100)
# check the binlog players is running
destination_master.wait_for_binlog_player_count(1)
# add values to source, make sure they're replicated
moving1_first_add1 = self._insert_values('moving1', 100)
staying1_first_add1 = self._insert_values('staying1', 100)
moving2_first_add1 = self._insert_values('moving2', 100)
self._check_values_timeout(destination_master, 'vt_destination_keyspace',
'moving1', moving1_first_add1, 100)
self._check_values_timeout(destination_master, 'vt_destination_keyspace',
'moving2', moving2_first_add1, 100)
# use the vtworker checker to compare the data
logging.debug("Running vtworker VerticalSplitDiff")
utils.run_vtworker(['-cell', 'test_nj', 'VerticalSplitDiff',
'destination_keyspace/0'], auto_log=True)
utils.run_vtctl(['ChangeSlaveType', source_rdonly1.tablet_alias, 'rdonly'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', source_rdonly2.tablet_alias, 'rdonly'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', destination_rdonly1.tablet_alias,
'rdonly'], auto_log=True)
utils.run_vtctl(['ChangeSlaveType', destination_rdonly2.tablet_alias,
'rdonly'], auto_log=True)
utils.pause("Good time to test vtworker for diffs")
# get status for destination master tablet, make sure we have it all
destination_master_status = destination_master.get_status()
self.assertIn('Binlog player state: Running', destination_master_status)
self.assertIn('moving.*', destination_master_status)
self.assertIn('<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status)
self.assertIn('</html>', destination_master_status)
# check query service is off on destination master, as filtered
# replication is enabled. Even health check should not interfere.
destination_master_vars = utils.get_vars(destination_master.port)
self.assertEqual(destination_master_vars['TabletStateName'], 'NOT_SERVING')
# check we can't migrate the master just yet
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'],
expect_fail=True)
# migrate rdonly only in test_ny cell, make sure nothing is migrated
# in test_nj
utils.run_vtctl(['MigrateServedFrom', '--cells=test_ny',
'destination_keyspace/0', 'rdonly'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly1, None)
self._check_blacklisted_tables(source_rdonly2, None)
# migrate test_nj only, using command line manual fix command,
# and restore it back.
keyspace_json = utils.run_vtctl_json(['GetKeyspace', 'destination_keyspace'])
self.assertEqual(keyspace_json['ServedFromMap']['rdonly']['Cells'],
['test_nj'])
utils.run_vtctl(['SetKeyspaceServedFrom', '-source=source_keyspace',
'-remove', '-cells=test_nj', 'destination_keyspace',
'rdonly'], auto_log=True)
keyspace_json = utils.run_vtctl_json(['GetKeyspace', 'destination_keyspace'])
self.assertFalse('rdonly' in keyspace_json['ServedFromMap'])
utils.run_vtctl(['SetKeyspaceServedFrom', '-source=source_keyspace',
'destination_keyspace', 'rdonly'],
auto_log=True)
keyspace_json = utils.run_vtctl_json(['GetKeyspace', 'destination_keyspace'])
self.assertEqual(keyspace_json['ServedFromMap']['rdonly']['Cells'],
None)
# now serve rdonly from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1'])
self._check_client_conn_redirection(
'source_keyspace', 'destination_keyspace', ['rdonly'],
['master', 'replica'], ['moving1', 'moving2'])
# then serve replica from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2'])
# move replica back and forth
utils.run_vtctl(['MigrateServedFrom', '-reverse',
'destination_keyspace/0', 'replica'], auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1'])
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1'])
self._check_client_conn_redirection(
'source_keyspace', 'destination_keyspace', ['replica', 'rdonly'],
['master'], ['moving1', 'moving2'])
# then serve master from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'],
auto_log=True)
self._check_srv_keyspace('')
self._check_blacklisted_tables(source_master, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_replica, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1'])
self._check_client_conn_redirection(
'source_keyspace', 'destination_keyspace',
['replica', 'rdonly', 'master'], [], ['moving1', 'moving2'])
# check 'vtctl SetShardTabletControl' command works as expected:
# clear the rdonly entry, re-add it, and then clear all entries.
utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0',
'rdonly'], auto_log=True)
shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0'])
self.assertNotIn('rdonly', shard_json['TabletControlMap'])
self.assertIn('replica', shard_json['TabletControlMap'])
self.assertIn('master', shard_json['TabletControlMap'])
utils.run_vtctl(['SetShardTabletControl', '--tables=moving.*,view1',
'source_keyspace/0', 'rdonly'], auto_log=True)
shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0'])
self.assertEqual(['moving.*', 'view1'],
shard_json['TabletControlMap']['rdonly']['BlacklistedTables'])
utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0',
'rdonly'], auto_log=True)
utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0',
'replica'], auto_log=True)
utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0',
'master'], auto_log=True)
shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0'])
self.assertEqual(None, shard_json['TabletControlMap'])
# check the binlog player is gone now
destination_master.wait_for_binlog_player_count(0)
# optional method to check the stats are correct
self._check_stats()
# kill everything
tablet.kill_tablets([source_master, source_replica, source_rdonly1,
source_rdonly2, destination_master,
destination_replica, destination_rdonly1,
destination_rdonly2])
def resolve_topology(self): topology.refresh_keyspace(self.zkocc_client, self.keyspace)
def test_vertical_split(self):
utils.run_vtctl(['CreateKeyspace', 'source_keyspace'])
utils.run_vtctl([
'CreateKeyspace', '--served_from',
'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace',
'destination_keyspace'
])
source_master.init_tablet('master', 'source_keyspace', '0')
source_replica.init_tablet('replica', 'source_keyspace', '0')
source_rdonly.init_tablet('rdonly', 'source_keyspace', '0')
# rebuild destination keyspace to make sure there is a serving
# graph entry, even though there is no tablet yet.
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
destination_master.init_tablet('master', 'destination_keyspace', '0')
destination_replica.init_tablet('replica', 'destination_keyspace', '0')
destination_rdonly.init_tablet('rdonly', 'destination_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
# create databases so vttablet can start behaving normally
for t in [source_master, source_replica, source_rdonly]:
t.create_db('vt_source_keyspace')
t.start_vttablet(wait_for_state=None)
destination_master.start_vttablet(wait_for_state=None,
target_tablet_type='replica')
for t in [destination_replica, destination_rdonly]:
t.start_vttablet(wait_for_state=None)
# wait for the tablets
for t in [source_master, source_replica, source_rdonly]:
t.wait_for_vttablet_state('SERVING')
for t in [destination_master, destination_replica, destination_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl([
'ReparentShard', '-force', 'source_keyspace/0',
source_master.tablet_alias
],
auto_log=True)
utils.run_vtctl([
'ReparentShard', '-force', 'destination_keyspace/0',
destination_master.tablet_alias
],
auto_log=True)
# read all the keyspaces, this will populate the topology cache.
self._populate_topo_cache()
# create the schema on the source keyspace, add some values
self._create_source_schema()
moving1_first = self._insert_values('moving1', 100)
moving2_first = self._insert_values('moving2', 100)
staying1_first = self._insert_values('staying1', 100)
staying2_first = self._insert_values('staying2', 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving1',
moving1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving2',
moving2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying1',
staying1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying2',
staying2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'view1',
moving1_first, 100)
if use_clone_worker:
# the worker will do everything. We test with source_reader_count=10
# (down from default=20) as connection pool is not big enough for 20.
# min_table_size_for_split is set to 1 as to force a split even on the
# small table we have.
utils.run_vtworker([
'--cell', 'test_nj', '--command_display_interval', '10ms',
'VerticalSplitClone', '--tables', 'moving.*,view1',
'--strategy', 'populateBlpCheckpoint', '--source_reader_count',
'10', '--min_table_size_for_split', '1',
'destination_keyspace/0'
],
auto_log=True)
else:
# take the snapshot for the split
utils.run_vtctl([
'MultiSnapshot', '--tables', 'moving.*,view1',
source_rdonly.tablet_alias
],
auto_log=True)
# perform the restore.
utils.run_vtctl([
'ShardMultiRestore', '--strategy', 'populateBlpCheckpoint',
'--tables', 'moving.*,view1', 'destination_keyspace/0',
source_rdonly.tablet_alias
],
auto_log=True)
topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace')
# check values are present
self._check_values(destination_master, 'vt_destination_keyspace',
'moving1', moving1_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace',
'moving2', moving2_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace',
'view1', moving1_first, 100)
# check the binlog players is running
destination_master.wait_for_binlog_player_count(1)
# add values to source, make sure they're replicated
moving1_first_add1 = self._insert_values('moving1', 100)
staying1_first_add1 = self._insert_values('staying1', 100)
moving2_first_add1 = self._insert_values('moving2', 100)
self._check_values_timeout(destination_master,
'vt_destination_keyspace', 'moving1',
moving1_first_add1, 100)
self._check_values_timeout(destination_master,
'vt_destination_keyspace', 'moving2',
moving2_first_add1, 100)
# use the vtworker checker to compare the data
logging.debug("Running vtworker VerticalSplitDiff")
utils.run_vtworker([
'-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0'
],
auto_log=True)
utils.run_vtctl(
['ChangeSlaveType', source_rdonly.tablet_alias, 'rdonly'],
auto_log=True)
utils.run_vtctl(
['ChangeSlaveType', destination_rdonly.tablet_alias, 'rdonly'],
auto_log=True)
utils.pause("Good time to test vtworker for diffs")
# get status for destination master tablet, make sure we have it all
destination_master_status = destination_master.get_status()
self.assertIn('Binlog player state: Running',
destination_master_status)
self.assertIn('moving.*', destination_master_status)
self.assertIn(
'<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>',
destination_master_status)
self.assertIn('</html>', destination_master_status)
# check query service is off on destination master, as filtered
# replication is enabled. Even health check should not interfere.
destination_master_vars = utils.get_vars(destination_master.port)
self.assertEqual(destination_master_vars['TabletStateName'],
'NOT_SERVING')
# check we can't migrate the master just yet
utils.run_vtctl(
['MigrateServedFrom', 'destination_keyspace/0', 'master'],
expect_fail=True)
# now serve rdonly from the destination shards
utils.run_vtctl(
['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1'])
self._check_client_conn_redirection('source_keyspace',
'destination_keyspace', ['rdonly'],
['master', 'replica'],
['moving1', 'moving2'])
# then serve replica from the destination shards
utils.run_vtctl(
['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1'])
self._check_client_conn_redirection('source_keyspace',
'destination_keyspace',
['replica', 'rdonly'], ['master'],
['moving1', 'moving2'])
# move replica back and forth
utils.run_vtctl([
'MigrateServedFrom', '-reverse', 'destination_keyspace/0',
'replica'
],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1'])
utils.run_vtctl(
['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1'])
self._check_client_conn_redirection('source_keyspace',
'destination_keyspace',
['replica', 'rdonly'], ['master'],
['moving1', 'moving2'])
# then serve master from the destination shards
utils.run_vtctl(
['MigrateServedFrom', 'destination_keyspace/0', 'master'],
auto_log=True)
self._check_srv_keyspace('')
self._check_blacklisted_tables(source_master, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_replica, ['moving.*', 'view1'])
self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1'])
self._check_client_conn_redirection('source_keyspace',
'destination_keyspace',
['replica', 'rdonly', 'master'],
[], ['moving1', 'moving2'])
# check 'vtctl SetShardBlacklistedTables' command works as expected:
# clear the rdonly entry, re-add it, and then clear all entries.
utils.run_vtctl(
['SetShardBlacklistedTables', 'source_keyspace/0', 'rdonly'],
auto_log=True)
shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0'])
self.assertNotIn('rdonly', shard_json['BlacklistedTablesMap'])
self.assertIn('replica', shard_json['BlacklistedTablesMap'])
self.assertIn('master', shard_json['BlacklistedTablesMap'])
utils.run_vtctl([
'SetShardBlacklistedTables', 'source_keyspace/0', 'rdonly',
'moving.*,view1'
],
auto_log=True)
shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0'])
self.assertEqual(['moving.*', 'view1'],
shard_json['BlacklistedTablesMap']['rdonly'])
utils.run_vtctl(
['SetShardBlacklistedTables', 'source_keyspace/0', 'rdonly'],
auto_log=True)
utils.run_vtctl(
['SetShardBlacklistedTables', 'source_keyspace/0', 'replica'],
auto_log=True)
utils.run_vtctl(
['SetShardBlacklistedTables', 'source_keyspace/0', 'master'],
auto_log=True)
shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0'])
self.assertEqual(None, shard_json['BlacklistedTablesMap'])
# check the binlog player is gone now
destination_master.wait_for_binlog_player_count(0)
# optional method to check the stats are correct
self._check_stats()
# kill everything
tablet.kill_tablets([
source_master, source_replica, source_rdonly, destination_master,
destination_replica, destination_rdonly
])
def refresh_keyspace(self, keyspace_name):
# This is so that keyspace can be refreshed.
time.sleep(self.throttle_sleep_interval)
topology.refresh_keyspace(self.vtgate_client, keyspace_name)
def test_vertical_split(self):
utils.run_vtctl(['CreateKeyspace', 'source_keyspace'])
utils.run_vtctl(['CreateKeyspace',
'--served-from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace',
'destination_keyspace'])
source_master.init_tablet('master', 'source_keyspace', '0')
source_replica.init_tablet('replica', 'source_keyspace', '0')
source_rdonly.init_tablet('rdonly', 'source_keyspace', '0')
# rebuild destination keyspace to make sure there is a serving
# graph entry, even though there is no tablet yet.
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
destination_master.init_tablet('master', 'destination_keyspace', '0')
destination_replica.init_tablet('replica', 'destination_keyspace', '0')
destination_rdonly.init_tablet('rdonly', 'destination_keyspace', '0')
utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(rdonly): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
# create databases so vttablet can start behaving normally
for t in [source_master, source_replica, source_rdonly]:
t.create_db('vt_source_keyspace')
t.start_vttablet(wait_for_state=None)
for t in [destination_master, destination_replica, destination_rdonly]:
t.start_vttablet(wait_for_state=None)
# wait for the tablets
for t in [source_master, source_replica, source_rdonly]:
t.wait_for_vttablet_state('SERVING')
for t in [destination_master, destination_replica, destination_rdonly]:
t.wait_for_vttablet_state('NOT_SERVING')
# reparent to make the tablets work
utils.run_vtctl(['ReparentShard', '-force', 'source_keyspace/0',
source_master.tablet_alias], auto_log=True)
utils.run_vtctl(['ReparentShard', '-force', 'destination_keyspace/0',
destination_master.tablet_alias], auto_log=True)
# read all the keyspaces, this will populate the topology cache.
self._populate_topo_cache()
# create the schema on the source keyspace, add some values
self._create_source_schema()
moving1_first = self._insert_values('moving1', 100)
moving2_first = self._insert_values('moving2', 100)
staying1_first = self._insert_values('staying1', 100)
staying2_first = self._insert_values('staying2', 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving1',
moving1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'moving2',
moving2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying1',
staying1_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'staying2',
staying2_first, 100)
self._check_values(source_master, 'vt_source_keyspace', 'view1',
moving1_first, 100)
# take the snapshot for the split
utils.run_vtctl(['MultiSnapshot',
'--tables', 'moving.*,view1',
source_rdonly.tablet_alias], auto_log=True)
# perform the restore.
utils.run_vtctl(['ShardMultiRestore',
'--strategy' ,'populateBlpCheckpoint',
'--tables', 'moving.*',
'destination_keyspace/0', source_rdonly.tablet_alias],
auto_log=True)
topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace')
# check values are present
self._check_values(destination_master, 'vt_destination_keyspace', 'moving1',
moving1_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace', 'moving2',
moving2_first, 100)
self._check_values(destination_master, 'vt_destination_keyspace', 'view1',
moving1_first, 100)
# check the binlog players is running
destination_master.wait_for_binlog_player_count(1)
# add values to source, make sure they're replicated
moving1_first_add1 = self._insert_values('moving1', 100)
staying1_first_add1 = self._insert_values('staying1', 100)
moving2_first_add1 = self._insert_values('moving2', 100)
self._check_values_timeout(destination_master, 'vt_destination_keyspace',
'moving1', moving1_first_add1, 100)
self._check_values_timeout(destination_master, 'vt_destination_keyspace',
'moving2', moving2_first_add1, 100)
# use the vtworker checker to compare the data
logging.debug("Running vtworker VerticalSplitDiff")
utils.run_vtworker(['-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', source_rdonly.tablet_alias, 'rdonly'],
auto_log=True)
utils.run_vtctl(['ChangeSlaveType', destination_rdonly.tablet_alias,
'rdonly'], auto_log=True)
utils.pause("Good time to test vtworker for diffs")
# get status for destination master tablet, make sure we have it all
destination_master_status = destination_master.get_status()
self.assertIn('Binlog player state: Running', destination_master_status)
self.assertIn('moving.*', destination_master_status)
self.assertIn('<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status)
self.assertIn('</html>', destination_master_status)
# check we can't migrate the master just yet
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'],
expect_fail=True)
# now serve rdonly from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['rdonly'], ['master', 'replica'], ['moving1', 'moving2'])
# then serve replica from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*'])
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2'])
# move replica back and forth
utils.run_vtctl(['MigrateServedFrom', '-reverse', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' +
'ServedFrom(replica): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, None)
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'],
auto_log=True)
self._check_srv_keyspace('ServedFrom(master): source_keyspace\n')
self._check_blacklisted_tables(source_master, None)
self._check_blacklisted_tables(source_replica, ['moving.*'])
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2'])
# then serve master from the destination shards
utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'],
auto_log=True)
self._check_srv_keyspace('')
self._check_blacklisted_tables(source_master, ['moving.*'])
self._check_blacklisted_tables(source_replica, ['moving.*'])
self._check_blacklisted_tables(source_rdonly, ['moving.*'])
self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly', 'master'], [], ['moving1', 'moving2'])
# check 'vtctl SetBlacklistedTables' command works as expected
utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias,
'moving.*,view1'], auto_log=True)
self._check_blacklisted_tables(source_master, ['moving.*', 'view1'])
utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias],
auto_log=True)
self._check_blacklisted_tables(source_master, None)
# check the binlog player is gone now
destination_master.wait_for_binlog_player_count(0)
# kill everything
tablet.kill_tablets([source_master, source_replica, source_rdonly,
destination_master, destination_replica,
destination_rdonly])