def test_topo_read_threshold(self): before_topo_rtt = get_topo_rtt() # Check original state. keyspace_obj = topology.get_keyspace("test_keyspace") self.assertNotEqual(keyspace_obj, None, "test_keyspace should be not None") self.assertEqual( keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64, "ShardingColumnType be %s" % keyrange_constants.KIT_UINT64, ) # Change the keyspace object. utils.run_vtctl( ["SetKeyspaceShardingInfo", "-force", "test_keyspace", "keyspace_id", keyrange_constants.KIT_BYTES] ) utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True) # sleep throttle interval and check values again. # the keyspace should have changed and also caused a rtt to topo server. time.sleep(self.keyspace_fetch_throttle) topology.refresh_keyspace(self.vtgate_client, "test_keyspace") keyspace_obj = topology.get_keyspace("test_keyspace") after_1st_clear = get_topo_rtt() self.assertEqual(after_1st_clear - before_topo_rtt, 1, "One additional round-trips to topo server") self.assertEqual( keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES, "ShardingColumnType be %s" % keyrange_constants.KIT_BYTES, ) # Refresh without sleeping for throttle time shouldn't cause additional rtt. topology.refresh_keyspace(self.vtgate_client, "test_keyspace") keyspace_obj = topology.get_keyspace("test_keyspace") after_2nd_clear = get_topo_rtt() self.assertEqual(after_2nd_clear - after_1st_clear, 0, "No additional round-trips to topo server")
def test_topo_read_threshold(self): before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt() # Check original state. keyspace_obj = topology.get_keyspace('test_keyspace') self.assertNotEqual(keyspace_obj, None, "test_keyspace should be not None") self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64, "ShardingColumnType be %s" % keyrange_constants.KIT_UINT64) # Change the keyspace object. utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id', keyrange_constants.KIT_BYTES]) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) # sleep throttle interval and check values again. # the keyspace should have changed and also caused a rtt to topo server. time.sleep(self.keyspace_fetch_throttle) topology.refresh_keyspace(self.vtgate_client, 'test_keyspace') keyspace_obj = topology.get_keyspace('test_keyspace') after_1st_clear = vtdb_logger.get_logger().get_topo_rtt() self.assertEqual(after_1st_clear - before_topo_rtt, 1, "One additional round-trips to topo server") self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES, "ShardingColumnType be %s" % keyrange_constants.KIT_BYTES) # Refresh without sleeping for throttle time shouldn't cause additional rtt. topology.refresh_keyspace(self.vtgate_client, 'test_keyspace') keyspace_obj = topology.get_keyspace('test_keyspace') after_2nd_clear = vtdb_logger.get_logger().get_topo_rtt() self.assertEqual(after_2nd_clear - after_1st_clear, 0, "No additional round-trips to topo server")
def test_topo_read_threshold(self): before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt() # Check original state. keyspace_obj = topology.get_keyspace('test_keyspace') self.assertNotEqual( keyspace_obj, None, 'test_keyspace should be not None') self.assertEqual( keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64, 'ShardingColumnType be %s' % keyrange_constants.KIT_UINT64) # Change the keyspace object. utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id', keyrange_constants.KIT_BYTES]) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) # sleep throttle interval and check values again. # the keyspace should have changed and also caused a rtt to topo server. time.sleep(self.keyspace_fetch_throttle) topology.refresh_keyspace(self.vtgate_client, 'test_keyspace') keyspace_obj = topology.get_keyspace('test_keyspace') after_1st_clear = vtdb_logger.get_logger().get_topo_rtt() self.assertEqual( after_1st_clear - before_topo_rtt, 1, 'One additional round-trips to topo server') self.assertEqual( keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES, 'ShardingColumnType be %s' % keyrange_constants.KIT_BYTES) # Refresh without sleeping for throttle time shouldn't cause # additional rtt. topology.refresh_keyspace(self.vtgate_client, 'test_keyspace') keyspace_obj = topology.get_keyspace('test_keyspace') after_2nd_clear = vtdb_logger.get_logger().get_topo_rtt() self.assertEqual( after_2nd_clear - after_1st_clear, 0, 'No additional round-trips to topo server')
def resolve_topology(self): topology.refresh_keyspace(self.topo_client, self.keyspace)
def test_vertical_split(self): utils.run_vtctl(['CreateKeyspace', 'source_keyspace']) utils.run_vtctl(['CreateKeyspace', '--served-from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace', 'destination_keyspace']) source_master.init_tablet('master', 'source_keyspace', '0') source_replica.init_tablet('replica', 'source_keyspace', '0') source_rdonly.init_tablet('rdonly', 'source_keyspace', '0') # rebuild destination keyspace to make sure there is a serving # graph entry, even though there is no tablet yet. utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') destination_master.init_tablet('master', 'destination_keyspace', '0') destination_replica.init_tablet('replica', 'destination_keyspace', '0') destination_rdonly.init_tablet('rdonly', 'destination_keyspace', '0') utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') # create databases so vttablet can start behaving normally for t in [source_master, source_replica, source_rdonly]: t.create_db('vt_source_keyspace') t.start_vttablet(wait_for_state=None) for t in [destination_master, destination_replica, destination_rdonly]: t.start_vttablet(wait_for_state=None) # wait for the tablets for t in [source_master, source_replica, source_rdonly]: t.wait_for_vttablet_state('SERVING') for t in [destination_master, destination_replica, destination_rdonly]: t.wait_for_vttablet_state('NOT_SERVING') # reparent to make the tablets work utils.run_vtctl(['ReparentShard', '-force', 'source_keyspace/0', source_master.tablet_alias], auto_log=True) utils.run_vtctl(['ReparentShard', '-force', 'destination_keyspace/0', destination_master.tablet_alias], auto_log=True) # read all the keyspaces, this will populate the topology cache. self._populate_topo_cache() # create the schema on the source keyspace, add some values self._create_source_schema() moving1_first = self._insert_values('moving1', 100) moving2_first = self._insert_values('moving2', 100) staying1_first = self._insert_values('staying1', 100) staying2_first = self._insert_values('staying2', 100) self._check_values(source_master, 'vt_source_keyspace', 'moving1', moving1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'moving2', moving2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying1', staying1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying2', staying2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'view1', moving1_first, 100) utils.pause("Before multisnapshot") # take the snapshot for the split utils.run_vtctl(['MultiSnapshot', '--tables', 'moving.*,view1', source_rdonly.tablet_alias], auto_log=True) # perform the restore. utils.run_vtctl(['ShardMultiRestore', '--strategy' ,'populateBlpCheckpoint', '--tables', 'moving.*', 'destination_keyspace/0', source_rdonly.tablet_alias], auto_log=True) topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace') # check values are present self._check_values(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'view1', moving1_first, 100) # check the binlog players is running destination_master.wait_for_binlog_player_count(1) # add values to source, make sure they're replicated moving1_first_add1 = self._insert_values('moving1', 100) staying1_first_add1 = self._insert_values('staying1', 100) moving2_first_add1 = self._insert_values('moving2', 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first_add1, 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first_add1, 100) # use the vtworker checker to compare the data logging.debug("Running vtworker VerticalSplitDiff") utils.run_vtworker(['-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', source_rdonly.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', destination_rdonly.tablet_alias, 'rdonly'], auto_log=True) utils.pause("Good time to test vtworker for diffs") # get status for destination master tablet, make sure we have it all destination_master_status = destination_master.get_status() self.assertIn('Binlog player state: Running', destination_master_status) self.assertIn('moving.*', destination_master_status) self.assertIn('<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status) self.assertIn('</html>', destination_master_status) # check we can't migrate the master just yet utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'], expect_fail=True) # now serve rdonly from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['rdonly'], ['master', 'replica'], ['moving1', 'moving2']) # then serve replica from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*']) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # move replica back and forth utils.run_vtctl(['MigrateServedFrom', '-reverse', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly, ['moving.*']) utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*']) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # then serve master from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'], auto_log=True) self._check_srv_keyspace('') self._check_blacklisted_tables(source_master, ['moving.*']) self._check_blacklisted_tables(source_replica, ['moving.*']) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly', 'master'], [], ['moving1', 'moving2']) # check 'vtctl SetBlacklistedTables' command works as expected utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias, 'moving.*,view1'], auto_log=True) self._check_blacklisted_tables(source_master, ['moving.*', 'view1']) utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias], auto_log=True) self._check_blacklisted_tables(source_master, None) # check the binlog player is gone now destination_master.wait_for_binlog_player_count(0) # optional method to check the stats are correct self._check_stats() # kill everything tablet.kill_tablets([source_master, source_replica, source_rdonly, destination_master, destination_replica, destination_rdonly])
def refresh_keyspace(self, keyspace_name): # This is so that keyspace can be refreshed. time.sleep(self.throttle_sleep_interval) topology.refresh_keyspace(self.vtgate_client, keyspace_name)
def test_vertical_split(self): utils.run_vtctl(['CreateKeyspace', 'source_keyspace']) utils.run_vtctl(['CreateKeyspace', '--served_from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace', 'destination_keyspace']) source_master.init_tablet('master', 'source_keyspace', '0') source_replica.init_tablet('replica', 'source_keyspace', '0') source_rdonly1.init_tablet('rdonly', 'source_keyspace', '0') source_rdonly2.init_tablet('rdonly', 'source_keyspace', '0') # rebuild destination keyspace to make sure there is a serving # graph entry, even though there is no tablet yet. utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') destination_master.init_tablet('master', 'destination_keyspace', '0') destination_replica.init_tablet('replica', 'destination_keyspace', '0') destination_rdonly1.init_tablet('rdonly', 'destination_keyspace', '0') destination_rdonly2.init_tablet('rdonly', 'destination_keyspace', '0') utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') # create databases so vttablet can start behaving normally for t in [source_master, source_replica, source_rdonly1, source_rdonly2]: t.create_db('vt_source_keyspace') t.start_vttablet(wait_for_state=None) destination_master.start_vttablet(wait_for_state=None, target_tablet_type='replica') for t in [destination_replica, destination_rdonly1, destination_rdonly2]: t.start_vttablet(wait_for_state=None) # wait for the tablets for t in [source_master, source_replica, source_rdonly1, source_rdonly2]: t.wait_for_vttablet_state('SERVING') for t in [destination_master, destination_replica, destination_rdonly1, destination_rdonly2]: t.wait_for_vttablet_state('NOT_SERVING') # reparent to make the tablets work utils.run_vtctl(['ReparentShard', '-force', 'source_keyspace/0', source_master.tablet_alias], auto_log=True) utils.run_vtctl(['ReparentShard', '-force', 'destination_keyspace/0', destination_master.tablet_alias], auto_log=True) # read all the keyspaces, this will populate the topology cache. self._populate_topo_cache() # create the schema on the source keyspace, add some values self._create_source_schema() moving1_first = self._insert_values('moving1', 100) moving2_first = self._insert_values('moving2', 100) staying1_first = self._insert_values('staying1', 100) staying2_first = self._insert_values('staying2', 100) self._check_values(source_master, 'vt_source_keyspace', 'moving1', moving1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'moving2', moving2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying1', staying1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying2', staying2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'view1', moving1_first, 100) # the worker will do everything. We test with source_reader_count=10 # (down from default=20) as connection pool is not big enough for 20. # min_table_size_for_split is set to 1 as to force a split even on the # small table we have. utils.run_vtctl(['CopySchemaShard', '--tables', 'moving.*,view1', source_rdonly1.tablet_alias, 'destination_keyspace/0'], auto_log=True) utils.run_vtworker(['--cell', 'test_nj', '--command_display_interval', '10ms', 'VerticalSplitClone', '--tables', 'moving.*,view1', '--strategy=-populate_blp_checkpoint', '--source_reader_count', '10', '--min_table_size_for_split', '1', 'destination_keyspace/0'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', source_rdonly1.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', source_rdonly2.tablet_alias, 'rdonly'], auto_log=True) topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace') # check values are present self._check_values(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'view1', moving1_first, 100) # check the binlog players is running destination_master.wait_for_binlog_player_count(1) # add values to source, make sure they're replicated moving1_first_add1 = self._insert_values('moving1', 100) staying1_first_add1 = self._insert_values('staying1', 100) moving2_first_add1 = self._insert_values('moving2', 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first_add1, 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first_add1, 100) # use the vtworker checker to compare the data logging.debug("Running vtworker VerticalSplitDiff") utils.run_vtworker(['-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', source_rdonly1.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', source_rdonly2.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', destination_rdonly1.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', destination_rdonly2.tablet_alias, 'rdonly'], auto_log=True) utils.pause("Good time to test vtworker for diffs") # get status for destination master tablet, make sure we have it all destination_master_status = destination_master.get_status() self.assertIn('Binlog player state: Running', destination_master_status) self.assertIn('moving.*', destination_master_status) self.assertIn('<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status) self.assertIn('</html>', destination_master_status) # check query service is off on destination master, as filtered # replication is enabled. Even health check should not interfere. destination_master_vars = utils.get_vars(destination_master.port) self.assertEqual(destination_master_vars['TabletStateName'], 'NOT_SERVING') # check we can't migrate the master just yet utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'], expect_fail=True) # migrate rdonly only in test_ny cell, make sure nothing is migrated # in test_nj utils.run_vtctl(['MigrateServedFrom', '--cells=test_ny', 'destination_keyspace/0', 'rdonly'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly1, None) self._check_blacklisted_tables(source_rdonly2, None) # migrate test_nj only, using command line manual fix command, # and restore it back. keyspace_json = utils.run_vtctl_json(['GetKeyspace', 'destination_keyspace']) self.assertEqual(keyspace_json['ServedFromMap']['rdonly']['Cells'], ['test_nj']) utils.run_vtctl(['SetKeyspaceServedFrom', '-source=source_keyspace', '-remove', '-cells=test_nj', 'destination_keyspace', 'rdonly'], auto_log=True) keyspace_json = utils.run_vtctl_json(['GetKeyspace', 'destination_keyspace']) self.assertFalse('rdonly' in keyspace_json['ServedFromMap']) utils.run_vtctl(['SetKeyspaceServedFrom', '-source=source_keyspace', 'destination_keyspace', 'rdonly'], auto_log=True) keyspace_json = utils.run_vtctl_json(['GetKeyspace', 'destination_keyspace']) self.assertEqual(keyspace_json['ServedFromMap']['rdonly']['Cells'], None) # now serve rdonly from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1']) self._check_client_conn_redirection( 'source_keyspace', 'destination_keyspace', ['rdonly'], ['master', 'replica'], ['moving1', 'moving2']) # then serve replica from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # move replica back and forth utils.run_vtctl(['MigrateServedFrom', '-reverse', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1']) utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1']) self._check_client_conn_redirection( 'source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # then serve master from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'], auto_log=True) self._check_srv_keyspace('') self._check_blacklisted_tables(source_master, ['moving.*', 'view1']) self._check_blacklisted_tables(source_replica, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly1, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly2, ['moving.*', 'view1']) self._check_client_conn_redirection( 'source_keyspace', 'destination_keyspace', ['replica', 'rdonly', 'master'], [], ['moving1', 'moving2']) # check 'vtctl SetShardTabletControl' command works as expected: # clear the rdonly entry, re-add it, and then clear all entries. utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0', 'rdonly'], auto_log=True) shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0']) self.assertNotIn('rdonly', shard_json['TabletControlMap']) self.assertIn('replica', shard_json['TabletControlMap']) self.assertIn('master', shard_json['TabletControlMap']) utils.run_vtctl(['SetShardTabletControl', '--tables=moving.*,view1', 'source_keyspace/0', 'rdonly'], auto_log=True) shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0']) self.assertEqual(['moving.*', 'view1'], shard_json['TabletControlMap']['rdonly']['BlacklistedTables']) utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0', 'rdonly'], auto_log=True) utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0', 'replica'], auto_log=True) utils.run_vtctl(['SetShardTabletControl', '--remove', 'source_keyspace/0', 'master'], auto_log=True) shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0']) self.assertEqual(None, shard_json['TabletControlMap']) # check the binlog player is gone now destination_master.wait_for_binlog_player_count(0) # optional method to check the stats are correct self._check_stats() # kill everything tablet.kill_tablets([source_master, source_replica, source_rdonly1, source_rdonly2, destination_master, destination_replica, destination_rdonly1, destination_rdonly2])
def resolve_topology(self): topology.refresh_keyspace(self.zkocc_client, self.keyspace)
def test_vertical_split(self): utils.run_vtctl(['CreateKeyspace', 'source_keyspace']) utils.run_vtctl([ 'CreateKeyspace', '--served_from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace', 'destination_keyspace' ]) source_master.init_tablet('master', 'source_keyspace', '0') source_replica.init_tablet('replica', 'source_keyspace', '0') source_rdonly.init_tablet('rdonly', 'source_keyspace', '0') # rebuild destination keyspace to make sure there is a serving # graph entry, even though there is no tablet yet. utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') destination_master.init_tablet('master', 'destination_keyspace', '0') destination_replica.init_tablet('replica', 'destination_keyspace', '0') destination_rdonly.init_tablet('rdonly', 'destination_keyspace', '0') utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') # create databases so vttablet can start behaving normally for t in [source_master, source_replica, source_rdonly]: t.create_db('vt_source_keyspace') t.start_vttablet(wait_for_state=None) destination_master.start_vttablet(wait_for_state=None, target_tablet_type='replica') for t in [destination_replica, destination_rdonly]: t.start_vttablet(wait_for_state=None) # wait for the tablets for t in [source_master, source_replica, source_rdonly]: t.wait_for_vttablet_state('SERVING') for t in [destination_master, destination_replica, destination_rdonly]: t.wait_for_vttablet_state('NOT_SERVING') # reparent to make the tablets work utils.run_vtctl([ 'ReparentShard', '-force', 'source_keyspace/0', source_master.tablet_alias ], auto_log=True) utils.run_vtctl([ 'ReparentShard', '-force', 'destination_keyspace/0', destination_master.tablet_alias ], auto_log=True) # read all the keyspaces, this will populate the topology cache. self._populate_topo_cache() # create the schema on the source keyspace, add some values self._create_source_schema() moving1_first = self._insert_values('moving1', 100) moving2_first = self._insert_values('moving2', 100) staying1_first = self._insert_values('staying1', 100) staying2_first = self._insert_values('staying2', 100) self._check_values(source_master, 'vt_source_keyspace', 'moving1', moving1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'moving2', moving2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying1', staying1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying2', staying2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'view1', moving1_first, 100) if use_clone_worker: # the worker will do everything. We test with source_reader_count=10 # (down from default=20) as connection pool is not big enough for 20. # min_table_size_for_split is set to 1 as to force a split even on the # small table we have. utils.run_vtworker([ '--cell', 'test_nj', '--command_display_interval', '10ms', 'VerticalSplitClone', '--tables', 'moving.*,view1', '--strategy', 'populateBlpCheckpoint', '--source_reader_count', '10', '--min_table_size_for_split', '1', 'destination_keyspace/0' ], auto_log=True) else: # take the snapshot for the split utils.run_vtctl([ 'MultiSnapshot', '--tables', 'moving.*,view1', source_rdonly.tablet_alias ], auto_log=True) # perform the restore. utils.run_vtctl([ 'ShardMultiRestore', '--strategy', 'populateBlpCheckpoint', '--tables', 'moving.*,view1', 'destination_keyspace/0', source_rdonly.tablet_alias ], auto_log=True) topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace') # check values are present self._check_values(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'view1', moving1_first, 100) # check the binlog players is running destination_master.wait_for_binlog_player_count(1) # add values to source, make sure they're replicated moving1_first_add1 = self._insert_values('moving1', 100) staying1_first_add1 = self._insert_values('staying1', 100) moving2_first_add1 = self._insert_values('moving2', 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first_add1, 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first_add1, 100) # use the vtworker checker to compare the data logging.debug("Running vtworker VerticalSplitDiff") utils.run_vtworker([ '-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0' ], auto_log=True) utils.run_vtctl( ['ChangeSlaveType', source_rdonly.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl( ['ChangeSlaveType', destination_rdonly.tablet_alias, 'rdonly'], auto_log=True) utils.pause("Good time to test vtworker for diffs") # get status for destination master tablet, make sure we have it all destination_master_status = destination_master.get_status() self.assertIn('Binlog player state: Running', destination_master_status) self.assertIn('moving.*', destination_master_status) self.assertIn( '<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status) self.assertIn('</html>', destination_master_status) # check query service is off on destination master, as filtered # replication is enabled. Even health check should not interfere. destination_master_vars = utils.get_vars(destination_master.port) self.assertEqual(destination_master_vars['TabletStateName'], 'NOT_SERVING') # check we can't migrate the master just yet utils.run_vtctl( ['MigrateServedFrom', 'destination_keyspace/0', 'master'], expect_fail=True) # now serve rdonly from the destination shards utils.run_vtctl( ['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['rdonly'], ['master', 'replica'], ['moving1', 'moving2']) # then serve replica from the destination shards utils.run_vtctl( ['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # move replica back and forth utils.run_vtctl([ 'MigrateServedFrom', '-reverse', 'destination_keyspace/0', 'replica' ], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1']) utils.run_vtctl( ['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # then serve master from the destination shards utils.run_vtctl( ['MigrateServedFrom', 'destination_keyspace/0', 'master'], auto_log=True) self._check_srv_keyspace('') self._check_blacklisted_tables(source_master, ['moving.*', 'view1']) self._check_blacklisted_tables(source_replica, ['moving.*', 'view1']) self._check_blacklisted_tables(source_rdonly, ['moving.*', 'view1']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly', 'master'], [], ['moving1', 'moving2']) # check 'vtctl SetShardBlacklistedTables' command works as expected: # clear the rdonly entry, re-add it, and then clear all entries. utils.run_vtctl( ['SetShardBlacklistedTables', 'source_keyspace/0', 'rdonly'], auto_log=True) shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0']) self.assertNotIn('rdonly', shard_json['BlacklistedTablesMap']) self.assertIn('replica', shard_json['BlacklistedTablesMap']) self.assertIn('master', shard_json['BlacklistedTablesMap']) utils.run_vtctl([ 'SetShardBlacklistedTables', 'source_keyspace/0', 'rdonly', 'moving.*,view1' ], auto_log=True) shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0']) self.assertEqual(['moving.*', 'view1'], shard_json['BlacklistedTablesMap']['rdonly']) utils.run_vtctl( ['SetShardBlacklistedTables', 'source_keyspace/0', 'rdonly'], auto_log=True) utils.run_vtctl( ['SetShardBlacklistedTables', 'source_keyspace/0', 'replica'], auto_log=True) utils.run_vtctl( ['SetShardBlacklistedTables', 'source_keyspace/0', 'master'], auto_log=True) shard_json = utils.run_vtctl_json(['GetShard', 'source_keyspace/0']) self.assertEqual(None, shard_json['BlacklistedTablesMap']) # check the binlog player is gone now destination_master.wait_for_binlog_player_count(0) # optional method to check the stats are correct self._check_stats() # kill everything tablet.kill_tablets([ source_master, source_replica, source_rdonly, destination_master, destination_replica, destination_rdonly ])
def test_vertical_split(self): utils.run_vtctl(['CreateKeyspace', 'source_keyspace']) utils.run_vtctl(['CreateKeyspace', '--served-from', 'master:source_keyspace,replica:source_keyspace,rdonly:source_keyspace', 'destination_keyspace']) source_master.init_tablet('master', 'source_keyspace', '0') source_replica.init_tablet('replica', 'source_keyspace', '0') source_rdonly.init_tablet('rdonly', 'source_keyspace', '0') # rebuild destination keyspace to make sure there is a serving # graph entry, even though there is no tablet yet. utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') destination_master.init_tablet('master', 'destination_keyspace', '0') destination_replica.init_tablet('replica', 'destination_keyspace', '0') destination_rdonly.init_tablet('rdonly', 'destination_keyspace', '0') utils.run_vtctl(['RebuildKeyspaceGraph', 'source_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'destination_keyspace'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(rdonly): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') # create databases so vttablet can start behaving normally for t in [source_master, source_replica, source_rdonly]: t.create_db('vt_source_keyspace') t.start_vttablet(wait_for_state=None) for t in [destination_master, destination_replica, destination_rdonly]: t.start_vttablet(wait_for_state=None) # wait for the tablets for t in [source_master, source_replica, source_rdonly]: t.wait_for_vttablet_state('SERVING') for t in [destination_master, destination_replica, destination_rdonly]: t.wait_for_vttablet_state('NOT_SERVING') # reparent to make the tablets work utils.run_vtctl(['ReparentShard', '-force', 'source_keyspace/0', source_master.tablet_alias], auto_log=True) utils.run_vtctl(['ReparentShard', '-force', 'destination_keyspace/0', destination_master.tablet_alias], auto_log=True) # read all the keyspaces, this will populate the topology cache. self._populate_topo_cache() # create the schema on the source keyspace, add some values self._create_source_schema() moving1_first = self._insert_values('moving1', 100) moving2_first = self._insert_values('moving2', 100) staying1_first = self._insert_values('staying1', 100) staying2_first = self._insert_values('staying2', 100) self._check_values(source_master, 'vt_source_keyspace', 'moving1', moving1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'moving2', moving2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying1', staying1_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'staying2', staying2_first, 100) self._check_values(source_master, 'vt_source_keyspace', 'view1', moving1_first, 100) # take the snapshot for the split utils.run_vtctl(['MultiSnapshot', '--tables', 'moving.*,view1', source_rdonly.tablet_alias], auto_log=True) # perform the restore. utils.run_vtctl(['ShardMultiRestore', '--strategy' ,'populateBlpCheckpoint', '--tables', 'moving.*', 'destination_keyspace/0', source_rdonly.tablet_alias], auto_log=True) topology.refresh_keyspace(self.vtgate_client, 'destination_keyspace') # check values are present self._check_values(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first, 100) self._check_values(destination_master, 'vt_destination_keyspace', 'view1', moving1_first, 100) # check the binlog players is running destination_master.wait_for_binlog_player_count(1) # add values to source, make sure they're replicated moving1_first_add1 = self._insert_values('moving1', 100) staying1_first_add1 = self._insert_values('staying1', 100) moving2_first_add1 = self._insert_values('moving2', 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving1', moving1_first_add1, 100) self._check_values_timeout(destination_master, 'vt_destination_keyspace', 'moving2', moving2_first_add1, 100) # use the vtworker checker to compare the data logging.debug("Running vtworker VerticalSplitDiff") utils.run_vtworker(['-cell', 'test_nj', 'VerticalSplitDiff', 'destination_keyspace/0'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', source_rdonly.tablet_alias, 'rdonly'], auto_log=True) utils.run_vtctl(['ChangeSlaveType', destination_rdonly.tablet_alias, 'rdonly'], auto_log=True) utils.pause("Good time to test vtworker for diffs") # get status for destination master tablet, make sure we have it all destination_master_status = destination_master.get_status() self.assertIn('Binlog player state: Running', destination_master_status) self.assertIn('moving.*', destination_master_status) self.assertIn('<td><b>All</b>: 1000<br><b>Query</b>: 700<br><b>Transaction</b>: 300<br></td>', destination_master_status) self.assertIn('</html>', destination_master_status) # check we can't migrate the master just yet utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'], expect_fail=True) # now serve rdonly from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'rdonly'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['rdonly'], ['master', 'replica'], ['moving1', 'moving2']) # then serve replica from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*']) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # move replica back and forth utils.run_vtctl(['MigrateServedFrom', '-reverse', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n' + 'ServedFrom(replica): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, None) self._check_blacklisted_tables(source_rdonly, ['moving.*']) utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'replica'], auto_log=True) self._check_srv_keyspace('ServedFrom(master): source_keyspace\n') self._check_blacklisted_tables(source_master, None) self._check_blacklisted_tables(source_replica, ['moving.*']) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly'], ['master'], ['moving1', 'moving2']) # then serve master from the destination shards utils.run_vtctl(['MigrateServedFrom', 'destination_keyspace/0', 'master'], auto_log=True) self._check_srv_keyspace('') self._check_blacklisted_tables(source_master, ['moving.*']) self._check_blacklisted_tables(source_replica, ['moving.*']) self._check_blacklisted_tables(source_rdonly, ['moving.*']) self._check_client_conn_redirection('source_keyspace', 'destination_keyspace', ['replica', 'rdonly', 'master'], [], ['moving1', 'moving2']) # check 'vtctl SetBlacklistedTables' command works as expected utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias, 'moving.*,view1'], auto_log=True) self._check_blacklisted_tables(source_master, ['moving.*', 'view1']) utils.run_vtctl(['SetBlacklistedTables', source_master.tablet_alias], auto_log=True) self._check_blacklisted_tables(source_master, None) # check the binlog player is gone now destination_master.wait_for_binlog_player_count(0) # kill everything tablet.kill_tablets([source_master, source_replica, source_rdonly, destination_master, destination_replica, destination_rdonly])