def setUpModule(): try: if environment.topo_server().flavor() == 'zookeeper': # this is a one-off test to make sure our zookeeper implementation # behaves with a server that is not DNS-resolveable environment.topo_server().setup(add_bad_host=True) else: environment.topo_server().setup() # start mysql instance external to the test setup_procs = [ tablet_62344.init_mysql(), tablet_62044.init_mysql(), ] utils.Vtctld().start() utils.wait_procs(setup_procs) except: tearDownModule() raise
def setUpModule(): try: environment.topo_server().setup() setup_procs = [ source_master.init_mysql(), source_replica.init_mysql(), source_rdonly1.init_mysql(), source_rdonly2.init_mysql(), destination_master.init_mysql(), destination_replica.init_mysql(), destination_rdonly1.init_mysql(), destination_rdonly2.init_mysql(), ] utils.Vtctld().start() utils.wait_procs(setup_procs) except: tearDownModule() raise
def setUpModule(): try: environment.topo_server().setup() _init_mysql(all_tablets) utils.run_vtctl(['CreateKeyspace', test_keyspace]) shard_0_master.init_tablet('master', test_keyspace, '0') shard_0_replica1.init_tablet('replica', test_keyspace, '0') shard_0_replica2.init_tablet('replica', test_keyspace, '0') shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0') shard_0_backup.init_tablet('backup', test_keyspace, '0') shard_1_master.init_tablet('master', test_keyspace, '1') shard_1_replica1.init_tablet('replica', test_keyspace, '1') # run checks now before we start the tablets utils.validate_topology() utils.Vtctld().start(enable_schema_change_dir=True) # create databases, start the tablets for t in initial_tablets: t.create_db(db_name) t.start_vttablet(wait_for_state=None) # wait for the tablets to start shard_0_master.wait_for_vttablet_state('SERVING') shard_0_replica1.wait_for_vttablet_state('NOT_SERVING') shard_0_replica2.wait_for_vttablet_state('NOT_SERVING') shard_0_rdonly.wait_for_vttablet_state('NOT_SERVING') shard_0_backup.wait_for_vttablet_state('NOT_SERVING') shard_1_master.wait_for_vttablet_state('SERVING') shard_1_replica1.wait_for_vttablet_state('NOT_SERVING') except Exception as setup_exception: # pylint: disable=broad-except try: tearDownModule() except Exception as e: # pylint: disable=broad-except logging.exception('Tearing down a failed setUpModule() failed: %s', e) raise setup_exception
def setUp(self): try: os.makedirs(environment.tmproot) except OSError: # directory already exists pass try: topo_flavor = environment.topo_server().flavor() environment.topo_server().setup() # start mysql instance external to the test setup_procs = [ self.replica.init_mysql(), self.master.init_mysql(), ] utils.Vtctld().start() logging.debug(utils.vtctld_connection) utils.wait_procs(setup_procs) for t in self.master, self.replica: t.create_db('vt_test_keyspace') self.master.init_tablet('replica', 'test_keyspace', '0', start=True) self.replica.init_tablet('replica', 'test_keyspace', '0', start=True) utils.run_vtctl([ 'InitShardMaster', '-force', 'test_keyspace/0', self.master.tablet_alias ]) self.master.mquery('vt_test_keyspace', _create_vt_insert_test) for t in [self.master, self.replica]: t.set_semi_sync_enabled(master=False, slave=False) except Exception as e: logging.exception(e) self.tearDown()
def setUpModule(): try: environment.topo_server().setup() utils.Vtctld().start() setup_procs = [ master_tablet.init_mysql(), replica_tablet.init_mysql(), ] utils.wait_procs(setup_procs) utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') master_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('vt_test_keyspace') except: tearDownModule() raise
def main(): parser = optparse.OptionParser(usage="usage: %prog [options]") utils.add_options(parser) (options, args) = parser.parse_args() options.debug = True utils.set_options(options) env = keyspace_util.TestEnv() vtgate_server = None try: environment.topo_server().setup() env.launch( "user", shards=["-80", "80-"], ddls=[ 'create table user(user_id bigint, name varchar(128), primary key(user_id))', 'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))', 'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))', 'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))', ], ) env.launch( "lookup", ddls=[ 'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))', 'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))', 'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))', ], ) utils.apply_vschema(vschema) vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s') utils.Vtctld().start() print "vtgate:", vtgate_port print "vtctld:", utils.vtctld.port utils.pause("the cluster is up, press enter to shut it down...") finally: utils.vtgate_kill(vtgate_server) env.teardown() utils.kill_sub_processes() utils.remove_tmp_files() environment.topo_server().teardown()
def setUpModule(): try: environment.topo_server().setup() # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()] utils.wait_procs(setup_procs) # start a vtctld so the vtctl insert commands are just RPCs, not forks utils.Vtctld().start() # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) utils.validate_topology() master_tablet.populate('vt_test_keyspace', create_vt_insert_test) replica_tablet.populate('vt_test_keyspace', create_vt_insert_test) utils.VtGate().start() master_tablet.start_vttablet(memcache=True, wait_for_state=None) replica_tablet.start_vttablet(memcache=True, wait_for_state=None) master_tablet.wait_for_vttablet_state('SERVING') replica_tablet.wait_for_vttablet_state('SERVING') utils.run_vtctl( ['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias], auto_log=True) utils.validate_topology() # restart the replica tablet so the stats are reset replica_tablet.kill_vttablet() replica_tablet.start_vttablet(memcache=True) except: tearDownModule() raise
def setUpModule(): try: if utils.options.xvfb: try: # This will be killed automatically by utils.kill_sub_processes() utils.run_bg(['Xvfb', ':15', '-ac']) os.environ['DISPLAY'] = ':15' except OSError as err: # Despite running in background, utils.run_bg() will throw immediately # if the Xvfb binary is not found. logging.error( "Can't start Xvfb (will try local DISPLAY instead): %s", err) environment.topo_server().setup() setup_procs = [t.init_mysql() for t in tablets] utils.Vtctld().start() utils.wait_procs(setup_procs) except: tearDownModule() raise
def setUp(self): self.master = tablet.Tablet() self.replica = tablet.Tablet() self.all_tablets = [self.master, self.replica] try: environment.topo_server().setup() setup_procs = [t.init_mysql() for t in self.all_tablets] utils.Vtctld().start() utils.wait_procs(setup_procs) utils.run_vtctl(['CreateKeyspace', KEYSPACE]) # Start tablets. db_name = 'vt_' + KEYSPACE for t in self.all_tablets: t.create_db(db_name) self.master.start_vttablet(wait_for_state=None, init_tablet_type='replica', init_keyspace=KEYSPACE, init_shard=SHARD, tablet_index=0) self.replica.start_vttablet(wait_for_state=None, init_tablet_type='replica', init_keyspace=KEYSPACE, init_shard=SHARD, tablet_index=1) for t in self.all_tablets: t.wait_for_vttablet_state('NOT_SERVING') # Reparent to choose an initial master and enable replication. utils.run_vtctl([ 'InitShardMaster', '-force', '%s/%s' % (KEYSPACE, SHARD), self.master.tablet_alias ]) # Create the schema. utils.run_vtctl(['ApplySchema', '-sql=' + SCHEMA, KEYSPACE]) # Start vtgate. utils.VtGate().start( extra_args=[ '-enable_vtgate_buffer', # Long timeout in case failover is slow. '-vtgate_buffer_window', '10m', '-vtgate_buffer_max_failover_duration', '10m', '-vtgate_buffer_min_time_between_failovers', '20m' ], tablets=self.all_tablets) # Insert two rows for the later threads (critical read, update). with utils.vtgate.write_transaction(keyspace=KEYSPACE, shards=[SHARD], tablet_type='master') as tx: tx.execute('INSERT INTO buffer (id, msg) VALUES (:id, :msg)', { 'id': CRITICAL_READ_ROW_ID, 'msg': 'critical read' }) tx.execute('INSERT INTO buffer (id, msg) VALUES (:id, :msg)', { 'id': UPDATE_ROW_ID, 'msg': 'update' }) except: self.tearDown() raise
def setUpModule(): global master_start_position try: environment.topo_server().setup() # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()] utils.wait_procs(setup_procs) # start a vtctld so the vtctl insert commands are just RPCs, not forks utils.Vtctld().start() # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0', tablet_index=0) replica_tablet.init_tablet('replica', 'test_keyspace', '0', tablet_index=1) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) utils.validate_topology() master_tablet.create_db('vt_test_keyspace') master_tablet.create_db('other_database') replica_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('other_database') master_tablet.start_vttablet(wait_for_state=None) replica_tablet.start_vttablet(wait_for_state=None) master_tablet.wait_for_vttablet_state('SERVING') replica_tablet.wait_for_vttablet_state('NOT_SERVING') for t in [master_tablet, replica_tablet]: t.reset_replication() utils.run_vtctl( ['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias], auto_log=True) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica') master_tablet.wait_for_vttablet_state('SERVING') replica_tablet.wait_for_vttablet_state('SERVING') # reset counter so tests don't assert tablet.Tablet.tablets_running = 0 master_start_position = _get_master_current_position() master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test) master_tablet.mquery('vt_test_keyspace', _create_vt_a) master_tablet.mquery('vt_test_keyspace', _create_vt_b) utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias]) utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) utils.run_vtctl(['RebuildVSchemaGraph']) utils.VtGate().start(tablets=[master_tablet, replica_tablet]) utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1) utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1) # Wait for the master and slave tablet's ReloadSchema to have worked. # Note we don't specify a keyspace name, there is only one, vschema # will just use that single keyspace. timeout = 10 while True: try: utils.vtgate.execute('select count(1) from vt_insert_test', tablet_type='master') utils.vtgate.execute('select count(1) from vt_insert_test', tablet_type='replica') break except protocols_flavor().client_error_exception_type(): logging.exception('query failed') timeout = utils.wait_step('slave tablet having correct schema', timeout) # also re-run ReloadSchema on slave, it case the first one # didn't get the replicated table. utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) except: tearDownModule() raise
def setUpModule(): try: environment.topo_server().setup() _init_mysql(tablets) utils.run_vtctl(['CreateKeyspace', test_keyspace]) shard_0_master.init_tablet('master', test_keyspace, '0') shard_0_replica1.init_tablet('replica', test_keyspace, '0') shard_0_replica2.init_tablet('replica', test_keyspace, '0') shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0') shard_0_backup.init_tablet('backup', test_keyspace, '0') shard_1_master.init_tablet('master', test_keyspace, '1') shard_1_replica1.init_tablet('replica', test_keyspace, '1') utils.run_vtctl(['RebuildKeyspaceGraph', test_keyspace], auto_log=True) # run checks now before we start the tablets utils.validate_topology() utils.Vtctld().start() # create databases, start the tablets for t in tablets: t.create_db(db_name) t.start_vttablet(wait_for_state=None) # wait for the tablets to start shard_0_master.wait_for_vttablet_state('SERVING') shard_0_replica1.wait_for_vttablet_state('SERVING') shard_0_replica2.wait_for_vttablet_state('SERVING') shard_0_rdonly.wait_for_vttablet_state('SERVING') shard_0_backup.wait_for_vttablet_state('NOT_SERVING') shard_1_master.wait_for_vttablet_state('SERVING') shard_1_replica1.wait_for_vttablet_state('SERVING') # make sure all replication is good for t in tablets: t.reset_replication() utils.run_vtctl([ 'InitShardMaster', test_keyspace + '/0', shard_0_master.tablet_alias ], auto_log=True) utils.run_vtctl([ 'InitShardMaster', test_keyspace + '/1', shard_1_master.tablet_alias ], auto_log=True) utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace]) # check after all tablets are here and replication is fixed utils.validate_topology(ping_tablets=True) except Exception as setup_exception: try: tearDownModule() except Exception as e: logging.exception("Tearing down a failed setUpModule() failed: %s", e) raise setup_exception
def setUpModule(): try: environment.topo_server().setup() setup_procs = [ src_master.init_mysql(), src_replica.init_mysql(), src_rdonly1.init_mysql(), src_rdonly2.init_mysql(), dst_master.init_mysql(), dst_replica.init_mysql(), ] utils.Vtctld().start() utils.wait_procs(setup_procs) # Set up binlog stream from shard 0 to shard 1. # Modeled after initial_sharding.py. utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id', keyrange_constants.KIT_UINT64]) src_master.init_tablet('master', 'test_keyspace', '0') src_replica.init_tablet('replica', 'test_keyspace', '0') src_rdonly1.init_tablet('rdonly', 'test_keyspace', '0') src_rdonly2.init_tablet('rdonly', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) for t in [src_master, src_replica, src_rdonly1, src_rdonly2]: t.create_db('vt_test_keyspace') t.start_vttablet(wait_for_state=None) for t in [src_master, src_replica, src_rdonly1, src_rdonly2]: t.wait_for_vttablet_state('SERVING') utils.run_vtctl(['InitShardMaster', 'test_keyspace/0', src_master.tablet_alias], auto_log=True) # Create schema logging.debug("Creating schema...") create_table = '''create table test_table( id bigint auto_increment, keyspace_id bigint(20) unsigned, msg varchar(64), primary key (id), index by_msg (msg) ) Engine=InnoDB''' utils.run_vtctl(['ApplySchema', '-sql=' + create_table, 'test_keyspace'], auto_log=True) # Create destination shard. dst_master.init_tablet('master', 'test_keyspace', '-') dst_replica.init_tablet('replica', 'test_keyspace', '-') dst_master.start_vttablet(wait_for_state='NOT_SERVING') dst_replica.start_vttablet(wait_for_state='NOT_SERVING') utils.run_vtctl(['InitShardMaster', 'test_keyspace/-', dst_master.tablet_alias], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) # copy the schema utils.run_vtctl(['CopySchemaShard', src_replica.tablet_alias, 'test_keyspace/-'], auto_log=True) # run the clone worked (this is a degenerate case, source and destination # both have the full keyrange. Happens to work correctly). logging.debug("Running the clone worker to start binlog stream...") utils.run_vtworker(['--cell', 'test_nj', 'SplitClone', '--strategy=-populate_blp_checkpoint', '--source_reader_count', '10', '--min_table_size_for_split', '1', 'test_keyspace/0'], auto_log=True) dst_master.wait_for_binlog_player_count(1) # Wait for dst_replica to be ready. dst_replica.wait_for_binlog_server_state("Enabled") except: tearDownModule() raise
def setUpModule(): worker.setUpModule() utils.Vtctld().start()
def setUpModule(): try: environment.topo_server().setup() setup_procs = [ src_master.init_mysql(), src_replica.init_mysql(), dst_master.init_mysql(), dst_replica.init_mysql(), ] utils.Vtctld().start() utils.wait_procs(setup_procs) # Set up binlog stream from shard 0 to shard 1. # Modeled after initial_sharding.py. utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) utils.run_vtctl([ 'SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id', keyrange_constants.KIT_UINT64 ]) src_master.init_tablet('master', 'test_keyspace', '0') src_replica.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) src_master.create_db('vt_test_keyspace') src_master.start_vttablet(wait_for_state=None) src_replica.create_db('vt_test_keyspace') src_replica.start_vttablet(wait_for_state=None) src_master.wait_for_vttablet_state('SERVING') src_replica.wait_for_vttablet_state('SERVING') utils.run_vtctl([ 'ReparentShard', '-force', 'test_keyspace/0', src_master.tablet_alias ], auto_log=True) # Create schema logging.debug("Creating schema...") create_table = '''create table test_table( id bigint auto_increment, keyspace_id bigint(20) unsigned, msg varchar(64), primary key (id), index by_msg (msg) ) Engine=InnoDB''' utils.run_vtctl([ 'ApplySchemaKeyspace', '-simple', '-sql=' + create_table, 'test_keyspace' ], auto_log=True) # Create destination shard. dst_master.init_tablet('master', 'test_keyspace', '1') dst_replica.init_tablet('replica', 'test_keyspace', '1') dst_master.start_vttablet(wait_for_state='NOT_SERVING') dst_replica.start_vttablet(wait_for_state='NOT_SERVING') utils.run_vtctl([ 'ReparentShard', '-force', 'test_keyspace/1', dst_master.tablet_alias ], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) # Start binlog stream from src_replica to dst_master. logging.debug("Starting binlog stream...") utils.run_vtctl(['MultiSnapshot', src_replica.tablet_alias], auto_log=True) src_replica.wait_for_binlog_server_state("Enabled") utils.run_vtctl([ 'ShardMultiRestore', '-strategy=-populate_blp_checkpoint', 'test_keyspace/1', src_replica.tablet_alias ], auto_log=True) dst_master.wait_for_binlog_player_count(1) # Wait for dst_replica to be ready. dst_replica.wait_for_binlog_server_state("Enabled") except: tearDownModule() raise
def setUpModule(): global master_start_position try: environment.topo_server().setup() # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()] utils.wait_procs(setup_procs) # start a vtctld so the vtctl insert commands are just RPCs, not forks utils.Vtctld().start() # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() master_tablet.create_db('vt_test_keyspace') master_tablet.create_db('other_database') replica_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('other_database') utils.VtGate().start() master_tablet.start_vttablet() replica_tablet.start_vttablet() utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias]) utils.check_db_read_write(master_tablet.tablet_uid) for t in [master_tablet, replica_tablet]: t.reset_replication() utils.run_vtctl( ['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias], auto_log=True) # reset counter so tests don't assert tablet.Tablet.tablets_running = 0 master_start_position = _get_master_current_position() master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test) master_tablet.mquery('vt_test_keyspace', _create_vt_a) master_tablet.mquery('vt_test_keyspace', _create_vt_b) utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias]) utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) # wait for the master and slave tablet's ReloadSchema to have worked timeout = 10 while True: try: master_tablet.execute('select count(1) from vt_insert_test') replica_tablet.execute('select count(1) from vt_insert_test') break except protocols_flavor().client_error_exception_type(): logging.exception('query failed') timeout = utils.wait_step('slave tablet having correct schema', timeout) # also re-run ReloadSchema on slave, it case the first one # didn't get the replicated table. utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) except: tearDownModule() raise
def setUpModule(): try: environment.topo_server().setup() setup_procs = [t.init_mysql() for t in all_tablets] utils.Vtctld().start() utils.wait_procs(setup_procs) # Set up binlog stream from shard 0 to shard 1. # Modeled after initial_sharding.py. utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) utils.run_vtctl([ 'SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id', keyrange_constants.KIT_UINT64 ]) src_master.init_tablet('replica', 'test_keyspace', '0') src_replica.init_tablet('replica', 'test_keyspace', '0') src_rdonly.init_tablet('rdonly', 'test_keyspace', '0') for t in [src_master, src_replica, src_rdonly]: t.start_vttablet(wait_for_state=None) for t in [src_master, src_replica, src_rdonly]: t.wait_for_vttablet_state('NOT_SERVING') utils.run_vtctl([ 'InitShardMaster', '-force', 'test_keyspace/0', src_master.tablet_alias ], auto_log=True) # Create schema logging.debug('Creating schema...') create_table = '''create table test_table( id bigint auto_increment, keyspace_id bigint(20) unsigned, msg varchar(64), primary key (id), index by_msg (msg) ) Engine=InnoDB''' utils.run_vtctl( ['ApplySchema', '-sql=' + create_table, 'test_keyspace'], auto_log=True) # run a health check on source replica so it responds to discovery # (for binlog players) and on the source rdonlys (for workers) utils.run_vtctl(['RunHealthCheck', src_replica.tablet_alias]) utils.run_vtctl(['RunHealthCheck', src_rdonly.tablet_alias]) # Create destination shard (won't be serving as there is no DB) dst_master.init_tablet('replica', 'test_keyspace', '-') dst_replica.init_tablet('replica', 'test_keyspace', '-') dst_rdonly.init_tablet('rdonly', 'test_keyspace', '-') dst_master.start_vttablet(wait_for_state='NOT_SERVING') dst_replica.start_vttablet(wait_for_state='NOT_SERVING') dst_rdonly.start_vttablet(wait_for_state='NOT_SERVING') utils.run_vtctl([ 'InitShardMaster', '-force', 'test_keyspace/-', dst_master.tablet_alias ], auto_log=True) # copy the schema utils.run_vtctl( ['CopySchemaShard', src_replica.tablet_alias, 'test_keyspace/-'], auto_log=True) # run the clone worker (this is a degenerate case, source and destination # both have the full keyrange. Happens to work correctly). logging.debug('Running the clone worker to start binlog stream...') utils.run_vtworker([ '--cell', 'test_nj', '--use_v3_resharding_mode=false', 'SplitClone', '--chunk_count', '10', '--min_rows_per_chunk', '1', '--min_healthy_rdonly_tablets', '1', 'test_keyspace/0' ], auto_log=True) dst_master.wait_for_binlog_player_count(1) # Wait for dst_replica to be ready. dst_replica.wait_for_binlog_server_state('Enabled') except: tearDownModule() raise