def setup_tablets(): # Start up a master mysql and vttablet utils.debug("Setting up tablets") utils.run_vtctl("CreateKeyspace test_keyspace") master_tablet.init_tablet("master", "test_keyspace", "0") utils.run_vtctl("RebuildShardGraph test_keyspace/0") utils.run_vtctl("RebuildKeyspaceGraph test_keyspace") utils.validate_topology() setup_schema() replica_tablet.create_db("vt_test_keyspace") # master_tablet.start_vttablet(auth=True) master_tablet.start_vttablet() replica_tablet.init_tablet("idle", "test_keyspace", start=True) snapshot_dir = os.path.join(utils.vtdataroot, "snapshot") utils.run("mkdir -p " + snapshot_dir) utils.run("chmod +w " + snapshot_dir) utils.run_vtctl("Clone -force %s %s" % (master_tablet.tablet_alias, replica_tablet.tablet_alias)) utils.run_vtctl("Ping test_nj-0000062344") utils.run_vtctl("SetReadWrite " + master_tablet.tablet_alias) utils.check_db_read_write(62344) utils.validate_topology() utils.run_vtctl("Ping test_nj-0000062345") # reset counter so tests don't assert tablet.Tablet.tablets_running = 0
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.validate_topology() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + ['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s']) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one _, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() tablet_62344.kill_vttablet()
def test_restart_during_action(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action utils.run_vtctl( ['-no-wait', 'Sleep', tablet_62344.tablet_alias, '15s'], stdout=utils.devnull) # ping blocks until the sleep finishes unless we have a schedule race action_path, _ = utils.run_vtctl( ['-no-wait', 'Ping', tablet_62344.tablet_alias], trap_output=True) action_path = action_path.strip() # kill agent leaving vtaction running tablet_62344.kill_vttablet() # restart agent tablet_62344.start_vttablet() # we expect this action with a short wait time to fail. this isn't the best # and has some potential for flakiness. utils.run_vtctl(['-wait-time', '2s', 'WaitForAction', action_path], expect_fail=True) # wait until the background sleep action is done, otherwise there will be # a leftover vtaction whose result may overwrite running actions # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug: # the zombie Sleep clobbers the Clone command in the following tests utils.run_vtctl(['-wait-time', '20s', 'WaitForAction', action_path], auto_log=True) if environment.topo_server_implementation == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def setup_tablets(): # Start up a master mysql and vttablet utils.debug("Setting up tablets") utils.run_vtctl('CreateKeyspace test_keyspace') master_tablet.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.run_vtctl('RebuildKeyspaceGraph test_keyspace') utils.validate_topology() setup_schema() replica_tablet.create_db('vt_test_keyspace') master_tablet.start_vttablet(memcache=True) replica_tablet.init_tablet('idle', 'test_keyspace', start=True, memcache=True) snapshot_dir = os.path.join(utils.vtdataroot, 'snapshot') utils.run("mkdir -p " + snapshot_dir) utils.run("chmod +w " + snapshot_dir) utils.run_vtctl('Clone -force %s %s' % (master_tablet.tablet_alias, replica_tablet.tablet_alias)) utils.run_vtctl('Ping test_nj-0000062344') utils.run_vtctl('SetReadWrite ' + master_tablet.tablet_alias) utils.check_db_read_write(62344) utils.validate_topology() utils.run_vtctl('Ping test_nj-0000062345') utils.run_vtctl('ChangeSlaveType test_nj-0000062345 replica')
def _test_sanity(self): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') utils.run_vtctl('CreateShard -force test_keyspace/0') tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl('RebuildKeyspaceGraph test_keyspace') utils.validate_topology() # if these statements don't run before the tablet it will wedge waiting for the # db to become accessible. this is more a bug than a feature. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away result, _ = utils.run_vtctl('Query test_nj test_keyspace "select * from vt_select_test"', trap_output=True) rows = result.splitlines() self.assertEqual(len(rows), 5, "expected 5 rows in vt_select_test: %s %s" % (str(rows), result)) # check Pings utils.run_vtctl('Ping ' + tablet_62344.tablet_alias) utils.run_vtctl('RpcPing ' + tablet_62344.tablet_alias) utils.validate_topology() utils.run_vtctl('ValidateKeyspace test_keyspace') # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl('ValidateShard -ping-tablets=false test_keyspace/0')
def setUpModule(): try: environment.topo_server().setup() _init_mysql(tablets) utils.run_vtctl(['CreateKeyspace', test_keyspace]) shard_0_master.init_tablet('master', test_keyspace, '0') shard_0_replica1.init_tablet('replica', test_keyspace, '0') shard_0_replica2.init_tablet('replica', test_keyspace, '0') shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0') shard_0_backup.init_tablet('backup', test_keyspace, '0') shard_1_master.init_tablet('master', test_keyspace, '1') shard_1_replica1.init_tablet('replica', test_keyspace, '1') # run checks now before we start the tablets utils.validate_topology() utils.Vtctld().start() # create databases, start the tablets for t in tablets: t.create_db(db_name) t.start_vttablet(wait_for_state=None) # wait for the tablets to start shard_0_master.wait_for_vttablet_state('SERVING') shard_0_replica1.wait_for_vttablet_state('SERVING') shard_0_replica2.wait_for_vttablet_state('SERVING') shard_0_rdonly.wait_for_vttablet_state('SERVING') shard_0_backup.wait_for_vttablet_state('NOT_SERVING') shard_1_master.wait_for_vttablet_state('SERVING') shard_1_replica1.wait_for_vttablet_state('SERVING') # make sure all replication is good for t in tablets: t.reset_replication() utils.run_vtctl([ 'InitShardMaster', test_keyspace + '/0', shard_0_master.tablet_alias ], auto_log=True) utils.run_vtctl([ 'InitShardMaster', test_keyspace + '/1', shard_1_master.tablet_alias ], auto_log=True) utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace]) # check after all tablets are here and replication is fixed utils.validate_topology(ping_tablets=True) except Exception as setup_exception: try: tearDownModule() except Exception as e: logging.exception('Tearing down a failed setUpModule() failed: %s', e) raise setup_exception
def _test_sanity(self): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') utils.run_vtctl('CreateShard -force test_keyspace/0') tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl('RebuildKeyspaceGraph test_keyspace') utils.validate_topology() # if these statements don't run before the tablet it will wedge waiting for the # db to become accessible. this is more a bug than a feature. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away result, _ = utils.run_vtctl( 'Query test_nj test_keyspace "select * from vt_select_test"', trap_output=True) rows = result.splitlines() self.assertEqual( len(rows), 5, "expected 5 rows in vt_select_test: %s %s" % (str(rows), result)) # check Pings utils.run_vtctl('Ping ' + tablet_62344.tablet_alias) utils.run_vtctl('RpcPing ' + tablet_62344.tablet_alias) utils.validate_topology() utils.run_vtctl('ValidateKeyspace test_keyspace') # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl('ValidateShard -ping-tablets=false test_keyspace/0')
def setup_tablets(): # Start up a master mysql and vttablet logging.debug("Setting up tablets") utils.run_vtctl('CreateKeyspace test_keyspace') master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() master_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('vt_test_keyspace') utils.run_vtctl('RebuildKeyspaceGraph test_keyspace') zkocc_server = utils.zkocc_start() master_tablet.start_vttablet() replica_tablet.start_vttablet() utils.run_vtctl('SetReadWrite ' + master_tablet.tablet_alias) utils.check_db_read_write(62344) for t in [master_tablet, replica_tablet]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + master_tablet.tablet_alias, auto_log=True) # reset counter so tests don't assert tablet.Tablet.tablets_running = 0 setup_schema() master_tablet.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0") replica_tablet.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
def _test_sanity(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace']) utils.run_vtctl(['createshard', '-force', 'test_keyspace/0']) tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) utils.validate_topology() srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') # if these statements don't run before the tablet it will wedge # waiting for the db to become accessible. this is more a bug than # a feature. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away result, _ = utils.run_vtctl(['Query', 'test_nj', 'test_keyspace', 'select * from vt_select_test'], mode=utils.VTCTL_VTCTL, trap_output=True) rows = result.splitlines() self.assertEqual(len(rows), 5, "expected 5 rows in vt_select_test: %s %s" % (str(rows), result)) # make sure direct dba queries work query_result = utils.run_vtctl_json(['ExecuteFetch', '-want_fields', tablet_62344.tablet_alias, 'select * from vt_test_keyspace.vt_select_test']) self.assertEqual(len(query_result['Rows']), 4, "expected 4 rows in vt_select_test: %s" % str(query_result)) self.assertEqual(len(query_result['Fields']), 2, "expected 2 fields in vt_select_test: %s" % str(query_result)) # check Pings utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias]) # Quickly check basic actions. utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias]) utils.check_db_read_write(62344) utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.validate_topology() utils.run_vtctl(['ValidateKeyspace', 'test_keyspace']) # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl(['ValidateShard', '-ping-tablets=false', 'test_keyspace/0']) srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.kill_vttablet() tablet_62344.init_tablet('idle') tablet_62344.scrap(force=True)
def setUpClass(klass): utils.run_vtctl("CreateKeyspace test_keyspace") shard_0_master.init_tablet("master", "test_keyspace", "-80") shard_0_replica.init_tablet("replica", "test_keyspace", "-80") shard_0_spare.init_tablet("spare", "test_keyspace", "-80") shard_1_master.init_tablet("master", "test_keyspace", "80-") shard_1_replica.init_tablet("replica", "test_keyspace", "80-") idle.init_tablet("idle") scrap.init_tablet("idle") utils.run_vtctl("RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*", auto_log=True) utils.run_vtctl("RebuildKeyspaceGraph /zk/global/vt/keyspaces/*", auto_log=True) for t in assigned: t.create_db("vt_test_keyspace") t.start_vttablet() for t in scrap, idle, shard_0_spare: t.start_vttablet(wait_for_state="NOT_SERVING") scrap.scrap() for t in [shard_0_master, shard_0_replica, shard_0_spare, shard_1_master, shard_1_replica, idle, scrap]: t.reset_replication() utils.run_vtctl("ReparentShard -force test_keyspace/-80 " + shard_0_master.tablet_alias, auto_log=True) utils.run_vtctl("ReparentShard -force test_keyspace/80- " + shard_1_master.tablet_alias, auto_log=True) # run checks now before we start the tablets utils.validate_topology()
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() srvShard = utils.run_vtctl_json( ['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + [ '-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s' ]) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one stdout, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() if environment.topo_server().flavor() == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def setUpClass(klass): utils.run_vtctl('CreateKeyspace test_keyspace') shard_0_master.init_tablet( 'master', 'test_keyspace', '-80') shard_0_replica.init_tablet('replica', 'test_keyspace', '-80') shard_0_spare.init_tablet('spare', 'test_keyspace', '-80') shard_1_master.init_tablet( 'master', 'test_keyspace', '80-') shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') idle.init_tablet('idle') scrap.init_tablet('idle') utils.run_vtctl('RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*', auto_log=True) utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True) for t in assigned: t.create_db('vt_test_keyspace') t.start_vttablet() for t in scrap, idle, shard_0_spare: t.start_vttablet(wait_for_state='NOT_SERVING') scrap.scrap() utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' + shard_0_master.tablet_alias, auto_log=True) utils.run_vtctl('ReparentShard -force test_keyspace/80- ' + shard_1_master.tablet_alias, auto_log=True) # run checks now before we start the tablets utils.validate_topology()
def test_multisnapshot_vtctl(): populate = sum([[ "insert into vt_insert_test_%s (msg) values ('test %s')" % (i, x) for x in xrange(4)] for i in range(6)], []) create = ['''create table vt_insert_test_%s ( id bigint auto_increment, msg varchar(64), primary key (id) ) Engine=InnoDB''' % i for i in range(6)] utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62344.populate('vt_test_keyspace', create, populate) tablet_62344.start_vttablet() utils.run_vtctl('MultiSnapshot --force --tables=vt_insert_test_1,vt_insert_test_2,vt_insert_test_3 --spec=-0000000000000003- %s id' % tablet_62344.tablet_alias) # if err != 0: # raise utils.TestError('mysqlctl multisnapshot failed') if os.path.exists(os.path.join(utils.vtdataroot, 'snapshot/vt_0000062344/data/vt_test_keyspace-,0000000000000003/vt_insert_test_4.0.csv.gz')): raise utils.TestError("Table vt_insert_test_4 wasn't supposed to be dumped.") for kr in 'vt_test_keyspace-,0000000000000003', 'vt_test_keyspace-0000000000000003,': path = os.path.join(utils.vtdataroot, 'snapshot/vt_0000062344/data/', kr, 'vt_insert_test_1.0.csv.gz') with gzip.open(path) as f: if len(f.readlines()) != 2: raise utils.TestError("Data looks wrong in %s" % path)
def _test_sanity(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace']) utils.run_vtctl(['createshard', '-force', 'test_keyspace/0']) tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) utils.validate_topology() srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') # if these statements don't run before the tablet it will wedge # waiting for the db to become accessible. this is more a bug than # a feature. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away result, _ = utils.run_vtctl(['Query', 'test_nj', 'test_keyspace', 'select * from vt_select_test'], mode=utils.VTCTL_VTCTL, trap_output=True) rows = result.splitlines() self.assertEqual(len(rows), 5, "expected 5 rows in vt_select_test: %s %s" % (str(rows), result)) # make sure direct dba queries work query_result = utils.run_vtctl_json(['ExecuteFetchAsDba', '-want_fields', tablet_62344.tablet_alias, 'select * from vt_test_keyspace.vt_select_test']) self.assertEqual(len(query_result['Rows']), 4, "expected 4 rows in vt_select_test: %s" % str(query_result)) self.assertEqual(len(query_result['Fields']), 2, "expected 2 fields in vt_select_test: %s" % str(query_result)) # check Ping / RefreshState utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) utils.run_vtctl(['RefreshState', tablet_62344.tablet_alias]) # Quickly check basic actions. utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias]) utils.check_db_read_write(62344) utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.validate_topology() utils.run_vtctl(['ValidateKeyspace', 'test_keyspace']) # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl(['ValidateShard', '-ping-tablets=false', 'test_keyspace/0']) srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.kill_vttablet() tablet_62344.init_tablet('idle') tablet_62344.scrap(force=True)
def run_test_reparent_lag_slave(shard_id='0'): utils.zk_wipe() utils.run_vtctl('CreateKeyspace -force test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True) tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True) tablet_41983.init_tablet('lag', 'test_keyspace', shard_id, start=True) # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id) utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' % (shard_id, tablet_62344.tablet_alias)) utils.validate_topology(ping_tablets=True) tablet_62344.create_db('vt_test_keyspace') tablet_62344.mquery('vt_test_keyspace', create_vt_insert_test) tablet_41983.mquery('', 'stop slave') for q in populate_vt_insert_test: tablet_62344.mquery('vt_test_keyspace', q, write=True) # Perform a graceful reparent operation. utils.run_vtctl('ReparentShard test_keyspace/%s %s' % (shard_id, tablet_62044.tablet_alias)) tablet_41983.mquery('', 'start slave') time.sleep(1) utils.pause("check orphan") utils.run_vtctl('ReparentTablet %s' % tablet_41983.tablet_alias) result = tablet_41983.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=1') if len(result) != 1: raise utils.TestError('expected 1 row from vt_insert_test', result) utils.pause("check lag reparent") tablet_62344.kill_vttablet() tablet_62044.kill_vttablet() tablet_41983.kill_vttablet() tablet_31981.kill_vttablet()
def test_reparent_cross_cell(self, shard_id='0'): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) # Create a few slaves for testing reparenting. Won't be healthy # as replication is not running. tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state('NOT_SERVING') # Force the slaves to reparent assuming that all the datasets are # identical. utils.run_vtctl([ 'InitShardMaster', '-force', 'test_keyspace/' + shard_id, tablet_62344.tablet_alias ], auto_log=True) utils.validate_topology(ping_tablets=True) self._check_master_tablet(tablet_62344) # Perform a graceful reparent operation to another cell. utils.run_vtctl([ 'PlannedReparentShard', '-keyspace_shard', 'test_keyspace/' + shard_id, '-new_master', tablet_31981.tablet_alias ], auto_log=True) utils.validate_topology() self._check_master_tablet(tablet_31981) tablet.kill_tablets( [tablet_62344, tablet_62044, tablet_41983, tablet_31981])
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server_flags() + environment.tablet_manager_protocol_flags() + environment.tabletconn_protocol_flags() + ['-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s']) bg = utils.run_bg(args) time.sleep(3) # try a frontend RpcPing that should timeout as the tablet is busy # running the other one stdout, stderr = utils.run_vtctl(['-wait-time', '3s', 'RpcPing', tablet_62344.tablet_alias], expect_fail=True) if 'Timeout waiting for' not in stderr: self.fail("didn't find the right error strings in failed RpcPing: " + stderr) # wait for the background vtctl bg.wait() if environment.topo_server_implementation == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def setUpClass(klass): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) utils.run_vtctl([ 'CreateKeyspace', '--served-from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace', 'redirected_keyspace' ]) shard_0_master.init_tablet('master', 'test_keyspace', '-80') shard_0_replica.init_tablet('spare', 'test_keyspace', '-80') shard_0_spare.init_tablet('spare', 'test_keyspace', '-80') shard_1_master.init_tablet('master', 'test_keyspace', '80-') shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') idle.init_tablet('idle') scrap.init_tablet('idle') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'], auto_log=True) for t in [shard_0_master, shard_1_master, shard_1_replica]: t.create_db('vt_test_keyspace') t.start_vttablet(extra_args=vtctld.process_args()) shard_0_replica.create_db('vt_test_keyspace') shard_0_replica.start_vttablet(extra_args=vtctld.process_args(), target_tablet_type='replica', wait_for_state='NOT_SERVING') for t in scrap, idle, shard_0_spare: t.start_vttablet(wait_for_state='NOT_SERVING', extra_args=vtctld.process_args()) scrap.scrap() for t in [ shard_0_master, shard_0_replica, shard_0_spare, shard_1_master, shard_1_replica, idle, scrap ]: t.reset_replication() utils.run_vtctl([ 'ReparentShard', '-force', 'test_keyspace/-80', shard_0_master.tablet_alias ], auto_log=True) utils.run_vtctl([ 'ReparentShard', '-force', 'test_keyspace/80-', shard_1_master.tablet_alias ], auto_log=True) shard_0_replica.wait_for_vttablet_state('SERVING') # run checks now before we start the tablets utils.validate_topology() # start a vtgate server too global vtgate_server, vtgate_port vtgate_server, vtgate_port = utils.vtgate_start( cache_ttl='0s', extra_args=vtctld.process_args())
def _test_reparent_slave_offline(self, shard_id='0'): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state('NOT_SERVING') # Force the slaves to reparent assuming that all the datasets are # identical. utils.run_vtctl([ 'InitShardMaster', '-force', 'test_keyspace/' + shard_id, tablet_62344.tablet_alias ]) utils.validate_topology(ping_tablets=True) self._check_master_tablet(tablet_62344) # Kill one tablet so we seem offline tablet_31981.kill_vttablet() # Perform a graceful reparent operation. utils.run_vtctl([ 'PlannedReparentShard', '-keyspace_shard', 'test_keyspace/' + shard_id, '-new_master', tablet_62044.tablet_alias ]) self._check_master_tablet(tablet_62044) tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983])
def test_restart_during_action(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0']) self.assertEqual(srvShard['MasterCell'], 'test_nj') tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action utils.run_vtctl(['-no-wait', 'Sleep', tablet_62344.tablet_alias, '15s'], stdout=utils.devnull) # ping blocks until the sleep finishes unless we have a schedule race action_path, _ = utils.run_vtctl(['-no-wait', 'Ping', tablet_62344.tablet_alias], trap_output=True) action_path = action_path.strip() # kill agent leaving vtaction running tablet_62344.kill_vttablet() # restart agent tablet_62344.start_vttablet() # we expect this action with a short wait time to fail. this isn't the best # and has some potential for flakiness. utils.run_vtctl(['-wait-time', '2s', 'WaitForAction', action_path], expect_fail=True) # wait until the background sleep action is done, otherwise there will be # a leftover vtaction whose result may overwrite running actions # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug: # the zombie Sleep clobbers the Clone command in the following tests utils.run_vtctl(['-wait-time', '20s', 'WaitForAction', action_path], auto_log=True) if environment.topo_server_implementation == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %u', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def setUpClass(cls): utils.run_vtctl([ 'CreateKeyspace', '--sharding_column_name', 'keyspace_id', '--sharding_column_type', 'uint64', 'test_keyspace' ]) utils.run_vtctl([ 'CreateKeyspace', '--served_from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace', 'redirected_keyspace' ]) shard_0_master.init_tablet('master', 'test_keyspace', '-80') shard_0_replica.init_tablet('spare', 'test_keyspace', '-80') shard_0_spare.init_tablet('spare', 'test_keyspace', '-80') shard_1_master.init_tablet('master', 'test_keyspace', '80-') shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'], auto_log=True) # start running all the tablets for t in [shard_0_master, shard_1_master, shard_1_replica]: t.create_db('vt_test_keyspace') t.start_vttablet(wait_for_state=None, extra_args=utils.vtctld.process_args()) shard_0_replica.create_db('vt_test_keyspace') shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(), target_tablet_type='replica', wait_for_state=None) shard_0_spare.start_vttablet(wait_for_state=None, extra_args=utils.vtctld.process_args()) # wait for the right states for t in [shard_0_master, shard_1_master, shard_1_replica]: t.wait_for_vttablet_state('SERVING') for t in [shard_0_replica, shard_0_spare]: t.wait_for_vttablet_state('NOT_SERVING') for t in [ shard_0_master, shard_0_replica, shard_0_spare, shard_1_master, shard_1_replica ]: t.reset_replication() utils.run_vtctl([ 'InitShardMaster', 'test_keyspace/-80', shard_0_master.tablet_alias ], auto_log=True) utils.run_vtctl([ 'InitShardMaster', 'test_keyspace/80-', shard_1_master.tablet_alias ], auto_log=True) shard_0_replica.wait_for_vttablet_state('SERVING') # run checks now utils.validate_topology()
def setUpModule(): global vtgate_server global vtgate_port global vtgate_socket_file global master_start_position try: environment.topo_server().setup() # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()] utils.wait_procs(setup_procs) # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() master_tablet.create_db('vt_test_keyspace') master_tablet.create_db('other_database') replica_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('other_database') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) vtgate_socket_file = environment.tmproot + '/vtgate.sock' vtgate_server, vtgate_port = utils.vtgate_start( socket_file=vtgate_socket_file) master_tablet.start_vttablet() replica_tablet.start_vttablet() utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias]) utils.check_db_read_write(master_tablet.tablet_uid) for t in [master_tablet, replica_tablet]: t.reset_replication() utils.run_vtctl([ 'ReparentShard', '-force', 'test_keyspace/0', master_tablet.tablet_alias ], auto_log=True) # reset counter so tests don't assert tablet.Tablet.tablets_running = 0 master_start_position = _get_master_current_position() master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test) master_tablet.mquery('vt_test_keyspace', _create_vt_a) master_tablet.mquery('vt_test_keyspace', _create_vt_b) utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias]) utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) except: tearDownModule() raise
def _test_reparent_slave_offline(self, shard_id='0'): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_41983.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.wait_for_vttablet_state("SERVING") # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id) utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' % (shard_id, tablet_62344.tablet_alias)) utils.validate_topology(ping_tablets=True) self._check_db_addr(shard_id, 'master', tablet_62344.port) # Kill one tablet so we seem offline tablet_31981.kill_vttablet() # Perform a graceful reparent operation. utils.run_vtctl('ReparentShard test_keyspace/%s %s' % (shard_id, tablet_62044.tablet_alias)) tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983])
def _test_sanity(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace']) utils.run_vtctl(['createshard', '-force', 'test_keyspace/0']) tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl( ['RebuildKeyspaceGraph', '-rebuild_srv_shards', 'test_keyspace']) utils.validate_topology() self._check_srv_shard() # if these statements don't run before the tablet it will wedge # waiting for the db to become accessible. this is more a bug than # a feature. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away qr = tablet_62344.execute('select * from vt_select_test') self.assertEqual(len(qr['Rows']), 4, 'expected 4 rows in vt_select_test: %s' % str(qr)) # make sure direct dba queries work query_result = utils.run_vtctl_json([ 'ExecuteFetchAsDba', tablet_62344.tablet_alias, 'select * from vt_test_keyspace.vt_select_test' ]) self.assertEqual( len(query_result['rows']), 4, 'expected 4 rows in vt_select_test: %s' % str(query_result)) self.assertEqual( len(query_result['fields']), 2, 'expected 2 fields in vt_select_test: %s' % str(query_result)) # check Ping / RefreshState utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) utils.run_vtctl(['RefreshState', tablet_62344.tablet_alias]) # Quickly check basic actions. utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias]) utils.check_db_read_write(62344) utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.validate_topology() utils.run_vtctl(['ValidateKeyspace', 'test_keyspace']) # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl( ['ValidateShard', '-ping-tablets=false', 'test_keyspace/0']) self._check_srv_shard() tablet_62344.kill_vttablet()
def test_multisnapshot_and_restore_vtctl(): tables = ["vt_insert_test", "vt_insert_test1"] create_template = """create table %s ( id bigint auto_increment, msg varchar(64), primary key (id) ) Engine=InnoDB""" insert_template = "insert into %s (id, msg) values (%s, 'test %s')" utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl("CreateKeyspace -force test_keyspace") # Start three tablets for three different shards. At this point the # sharding schema is not really important, as long as it is # consistent. new_spec = "-0000000000000028-" old_tablets = [tablet_62044, tablet_41983, tablet_31981] for i, tablet in enumerate(old_tablets): tablet.init_tablet("master", "test_keyspace", str(i)) utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i) utils.validate_topology() for i, tablet in enumerate(old_tablets): tablet.populate( "vt_test_keyspace", [create_template % table for table in tables], sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []), ) tablet.start_vttablet() utils.run_vtctl( "MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id" % (new_spec, tablet.tablet_alias), trap_output=True, ) utils.run_vtctl("CreateKeyspace -force test_keyspace_new") tablet_62344.init_tablet("master", "test_keyspace_new", "-0000000000000028", dbname="not_vt_test_keyspace") utils.run_vtctl("RebuildShardGraph test_keyspace_new/-0000000000000028") utils.validate_topology() tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace") tablet_62344.start_vttablet(wait_for_state="CONNECTING") # db not created # 0x28 = 40 source_aliases = " ".join(t.tablet_alias for t in old_tablets) utils.run_vtctl( "MultiRestore %s %s" % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True ) time.sleep(1) for table in tables: rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table) if len(rows) == 0: raise utils.TestError("There are no rows in the restored database.") for row in rows: if row[0] > 32: raise utils.TestError("Bad row: %s" % row) for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344: tablet.kill_vttablet()
def run_test_mysqlctl_split(): utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62344.populate('vt_test_keyspace', create_vt_insert_test, populate_vt_insert_test) tablet_62344.start_vttablet() err = tablet_62344.mysqlctl('-port %u -mysql-port %u partialsnapshot -end=0000000000000003 vt_test_keyspace id' % (tablet_62344.port, tablet_62344.mysql_port)).wait() if err != 0: raise utils.TestError('mysqlctl partialsnapshot failed') utils.pause("partialsnapshot finished") tablet_62044.mquery('', 'stop slave') tablet_62044.create_db('vt_test_keyspace') call(["touch", "/tmp/vtSimulateFetchFailures"]) err = tablet_62044.mysqlctl('-port %u -mysql-port %u partialrestore %s/snapshot/vt_0000062344/data/vt_test_keyspace-,0000000000000003/partial_snapshot_manifest.json' % (tablet_62044.port, tablet_62044.mysql_port, utils.vtdataroot)).wait() if err != 0: raise utils.TestError('mysqlctl partialrestore failed') tablet_62044.assert_table_count('vt_test_keyspace', 'vt_insert_test', 2) # change/add two values on the master, one in range, one out of range, make # sure the right one propagate and not the other utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias) tablet_62344.mquery('vt_test_keyspace', "insert into vt_insert_test (id, msg) values (5, 'test should not propagate')", write=True) tablet_62344.mquery('vt_test_keyspace', "update vt_insert_test set msg='test should propagate' where id=2", write=True) utils.pause("look at db now!") # wait until value that should have been changed is here timeout = 10 while timeout > 0: result = tablet_62044.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=2') if result[0][0] == "test should propagate": break timeout -= 1 time.sleep(1) if timeout == 0: raise utils.TestError("expected propagation to happen", result) # test value that should not propagate # this part is disabled now, as the replication pruning is only enabled # for row-based replication, but the mysql server is statement based. # will re-enable once we get statement-based pruning patch into mysql. # tablet_62044.assert_table_count('vt_test_keyspace', 'vt_insert_test', 0, 'where id=5') tablet_62344.kill_vttablet()
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(["CreateKeyspace", "test_keyspace"]) tablet_62344.init_tablet("master", "test_keyspace", "0") utils.run_vtctl(["RebuildShardGraph", "test_keyspace/0"]) utils.validate_topology() self._check_srv_shard() tablet_62344.create_db("vt_test_keyspace") tablet_62344.start_vttablet() utils.run_vtctl(["Ping", tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = ( environment.binary_args("vtctl") + environment.topo_server().flags() + [ "-tablet_manager_protocol", protocols_flavor().tablet_manager_protocol(), "-tablet_protocol", protocols_flavor().tabletconn_protocol(), "-log_dir", environment.vtlogroot, "Sleep", tablet_62344.tablet_alias, "10s", ] ) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one _, stderr = utils.run_vtctl(["-wait-time", "3s", "RefreshState", tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() if environment.topo_server().flavor() == "zookeeper": # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s", v) # then the Zookeeper connections if v["ZkCachedConn"]["test_nj"] != "Connected": self.fail("invalid zk test_nj state: %s" % v["ZkCachedConn"]["test_nj"]) if v["ZkCachedConn"]["global"] != "Connected": self.fail("invalid zk global state: %s" % v["ZkCachedConn"]["global"]) if v["TabletType"] != "master": self.fail("TabletType not exported correctly") tablet_62344.kill_vttablet()
def _test_sanity(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace']) utils.run_vtctl(['createshard', '-force', 'test_keyspace/0']) tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl( ['RebuildKeyspaceGraph', '-rebuild_srv_shards', 'test_keyspace']) utils.validate_topology() self._check_srv_shard() # if these statements don't run before the tablet it will wedge # waiting for the db to become accessible. this is more a bug than # a feature. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away qr = tablet_62344.execute('select * from vt_select_test') self.assertEqual(len(qr['rows']), 4, 'expected 4 rows in vt_select_test: %s' % str(qr)) # make sure direct dba queries work query_result = utils.run_vtctl_json( ['ExecuteFetchAsDba', '-json', tablet_62344.tablet_alias, 'select * from vt_test_keyspace.vt_select_test']) self.assertEqual( len(query_result['rows']), 4, 'expected 4 rows in vt_select_test: %s' % str(query_result)) self.assertEqual( len(query_result['fields']), 2, 'expected 2 fields in vt_select_test: %s' % str(query_result)) # check Ping / RefreshState utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) utils.run_vtctl(['RefreshState', tablet_62344.tablet_alias]) # Quickly check basic actions. utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias]) utils.check_db_read_write(62344) utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias]) utils.wait_db_read_only(62344) utils.validate_topology() utils.run_vtctl(['ValidateKeyspace', 'test_keyspace']) # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl(['ValidateShard', '-ping-tablets=false', 'test_keyspace/0']) self._check_srv_shard() tablet_62344.kill_vttablet()
def setUpClass(klass): utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) utils.run_vtctl(['CreateKeyspace', '--served-from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace', 'redirected_keyspace']) shard_0_master.init_tablet( 'master', 'test_keyspace', '-80') shard_0_replica.init_tablet('spare', 'test_keyspace', '-80') shard_0_spare.init_tablet( 'spare', 'test_keyspace', '-80') shard_1_master.init_tablet( 'master', 'test_keyspace', '80-') shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') idle.init_tablet('idle') scrap.init_tablet('idle') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'], auto_log=True) # start running all the tablets for t in [shard_0_master, shard_1_master, shard_1_replica]: t.create_db('vt_test_keyspace') t.start_vttablet(wait_for_state=None, extra_args=utils.vtctld.process_args()) shard_0_replica.create_db('vt_test_keyspace') shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(), target_tablet_type='replica', wait_for_state=None) for t in scrap, idle, shard_0_spare: t.start_vttablet(wait_for_state=None, extra_args=utils.vtctld.process_args()) # wait for the right states for t in [shard_0_master, shard_1_master, shard_1_replica]: t.wait_for_vttablet_state('SERVING') for t in [scrap, idle, shard_0_replica, shard_0_spare]: t.wait_for_vttablet_state('NOT_SERVING') scrap.scrap() for t in [shard_0_master, shard_0_replica, shard_0_spare, shard_1_master, shard_1_replica, idle, scrap]: t.reset_replication() utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80', shard_0_master.tablet_alias], auto_log=True) utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-', shard_1_master.tablet_alias], auto_log=True) shard_0_replica.wait_for_vttablet_state('SERVING') # run checks now before we start the tablets utils.validate_topology() # start a vtgate server too global vtgate_server, vtgate_port vtgate_server, vtgate_port = utils.vtgate_start( cache_ttl='0s', extra_args=utils.vtctld.process_args())
def test_reparent_lag_slave(self, shard_id='0'): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True, wait_for_start=False) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False) tablet_41983.init_tablet('lag', 'test_keyspace', shard_id, start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_31981]: t.wait_for_vttablet_state("SERVING") tablet_41983.wait_for_vttablet_state("NOT_SERVING") # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id) utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' % (shard_id, tablet_62344.tablet_alias)) utils.validate_topology(ping_tablets=True) tablet_62344.create_db('vt_test_keyspace') tablet_62344.mquery('vt_test_keyspace', self._create_vt_insert_test) tablet_41983.mquery('', 'stop slave') for q in self._populate_vt_insert_test: tablet_62344.mquery('vt_test_keyspace', q, write=True) # Perform a graceful reparent operation. utils.run_vtctl('ReparentShard test_keyspace/%s %s' % (shard_id, tablet_62044.tablet_alias)) tablet_41983.mquery('', 'start slave') time.sleep(1) utils.pause("check orphan") utils.run_vtctl('ReparentTablet %s' % tablet_41983.tablet_alias) result = tablet_41983.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=1') if len(result) != 1: self.fail('expected 1 row from vt_insert_test: %s' % str(result)) utils.pause("check lag reparent") tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983, tablet_31981])
def test_actions_and_timeouts(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() self._check_srv_shard() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl(['Ping', tablet_62344.tablet_alias]) # schedule long action in the background, sleep a little bit to make sure # it started to run args = (environment.binary_args('vtctl') + environment.topo_server().flags() + ['-tablet_manager_protocol', protocols_flavor().tablet_manager_protocol(), '-tablet_protocol', protocols_flavor().tabletconn_protocol(), '-log_dir', environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias, '10s']) bg = utils.run_bg(args) time.sleep(3) # try a frontend RefreshState that should timeout as the tablet is busy # running the other one stdout, stderr = utils.run_vtctl( ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias], expect_fail=True) self.assertIn(protocols_flavor().rpc_timeout_message(), stderr) # wait for the background vtctl bg.wait() if environment.topo_server().flavor() == 'zookeeper': # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug('vars: %s', v) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': self.fail('invalid zk test_nj state: %s' % v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': self.fail('invalid zk global state: %s' % v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: self.fail('not enough time in Connected state: %d', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': self.fail('TabletType not exported correctly') tablet_62344.kill_vttablet()
def setUpModule(): global vtgate_server global vtgate_port global vtgate_socket_file global master_start_position try: environment.topo_server_setup() # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql() ] utils.wait_procs(setup_procs) # Start up a master mysql and vttablet logging.debug("Setting up tablets") utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() master_tablet.create_db('vt_test_keyspace') master_tablet.create_db('other_database') replica_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('other_database') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) vtgate_socket_file = environment.tmproot + '/vtgate.sock' vtgate_server, vtgate_port = utils.vtgate_start(socket_file=vtgate_socket_file) master_tablet.start_vttablet() replica_tablet.start_vttablet() utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias]) utils.check_db_read_write(master_tablet.tablet_uid) for t in [master_tablet, replica_tablet]: t.reset_replication() utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0', master_tablet.tablet_alias], auto_log=True) # reset counter so tests don't assert tablet.Tablet.tablets_running = 0 master_start_position = _get_master_current_position() master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test) master_tablet.mquery('vt_test_keyspace', _create_vt_a) master_tablet.mquery('vt_test_keyspace', _create_vt_b) utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias]) utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) except: tearDownModule() raise
def test_vtgate(self): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') utils.run_vtctl('CreateShard -force test_keyspace/0') tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.run_vtctl('RebuildKeyspaceGraph test_keyspace') utils.validate_topology() # if these statements don't run before the tablet it will wedge waiting for the # db to become accessible. this is more a bug than a feature. tablet_62344.mquery("", ["set global read_only = off"]) tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet() gate_proc = utils.vtgate_start() try: conn = vtgate.connect("localhost:%s"%(utils.vtgate_port_base), "master", "test_keyspace", "0", 2.0) # _execute (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {}) self.assertEqual(count, 4, "want 4, got %d" % (count)) self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields))) # _stream_execute (result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {}) self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields))) count = 0 while 1: r = conn._stream_next() if not r: break count += 1 self.assertEqual(count, 4, "want 4, got %d" % (count)) # begin-rollback conn.begin() conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"}) conn.rollback() (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {}) self.assertEqual(count, 4, "want 4, got %d" % (count)) # begin-commit conn.begin() conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"}) conn.commit() (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {}) self.assertEqual(count, 5, "want 5, got %d" % (count)) # close conn.close() finally: utils.vtgate_kill(gate_proc) tablet_62344.kill_vttablet()
def run_test_reparent_lag_slave(shard_id="0"): utils.zk_wipe() utils.run_vtctl("CreateKeyspace -force test_keyspace") # create the database so vttablets start, as they are serving tablet_62344.create_db("vt_test_keyspace") tablet_62044.create_db("vt_test_keyspace") tablet_41983.create_db("vt_test_keyspace") tablet_31981.create_db("vt_test_keyspace") # Start up a master mysql and vttablet tablet_62344.init_tablet("master", "test_keyspace", shard_id, start=True) # Create a few slaves for testing reparenting. tablet_62044.init_tablet("replica", "test_keyspace", shard_id, start=True) tablet_31981.init_tablet("replica", "test_keyspace", shard_id, start=True) tablet_41983.init_tablet("lag", "test_keyspace", shard_id, start=True) # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl("RebuildShardGraph test_keyspace/" + shard_id) utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]: t.reset_replication() utils.run_vtctl("ReparentShard -force test_keyspace/%s %s" % (shard_id, tablet_62344.tablet_alias)) utils.validate_topology(ping_tablets=True) tablet_62344.create_db("vt_test_keyspace") tablet_62344.mquery("vt_test_keyspace", create_vt_insert_test) tablet_41983.mquery("", "stop slave") for q in populate_vt_insert_test: tablet_62344.mquery("vt_test_keyspace", q, write=True) # Perform a graceful reparent operation. utils.run_vtctl("ReparentShard test_keyspace/%s %s" % (shard_id, tablet_62044.tablet_alias)) tablet_41983.mquery("", "start slave") time.sleep(1) utils.pause("check orphan") utils.run_vtctl("ReparentTablet %s" % tablet_41983.tablet_alias) result = tablet_41983.mquery("vt_test_keyspace", "select msg from vt_insert_test where id=1") if len(result) != 1: raise utils.TestError("expected 1 row from vt_insert_test", result) utils.pause("check lag reparent") tablet_62344.kill_vttablet() tablet_62044.kill_vttablet() tablet_41983.kill_vttablet() tablet_31981.kill_vttablet()
def setUpModule(): try: environment.topo_server().setup() _init_mysql(tablets) utils.run_vtctl(['CreateKeyspace', test_keyspace]) shard_0_master.init_tablet( 'master', test_keyspace, '0') shard_0_replica1.init_tablet('replica', test_keyspace, '0') shard_0_replica2.init_tablet('replica', test_keyspace, '0') shard_0_rdonly.init_tablet( 'rdonly', test_keyspace, '0') shard_0_backup.init_tablet( 'backup', test_keyspace, '0') shard_1_master.init_tablet( 'master', test_keyspace, '1') shard_1_replica1.init_tablet('replica', test_keyspace, '1') utils.run_vtctl(['RebuildKeyspaceGraph', test_keyspace], auto_log=True) # run checks now before we start the tablets utils.validate_topology() utils.Vtctld().start() # create databases, start the tablets for t in tablets: t.create_db(db_name) t.start_vttablet(wait_for_state=None) # wait for the tablets to start shard_0_master.wait_for_vttablet_state('SERVING') shard_0_replica1.wait_for_vttablet_state('SERVING') shard_0_replica2.wait_for_vttablet_state('SERVING') shard_0_rdonly.wait_for_vttablet_state('SERVING') shard_0_backup.wait_for_vttablet_state('NOT_SERVING') shard_1_master.wait_for_vttablet_state('SERVING') shard_1_replica1.wait_for_vttablet_state('SERVING') # make sure all replication is good for t in tablets: t.reset_replication() utils.run_vtctl(['InitShardMaster', test_keyspace+'/0', shard_0_master.tablet_alias], auto_log=True) utils.run_vtctl(['InitShardMaster', test_keyspace+'/1', shard_1_master.tablet_alias], auto_log=True) utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace]) # check after all tablets are here and replication is fixed utils.validate_topology(ping_tablets=True) except Exception as setup_exception: try: tearDownModule() except Exception as e: logging.exception("Tearing down a failed setUpModule() failed: %s", e) raise setup_exception
def _test_vtctl_clone(self, server_mode): if server_mode: clone_flags = '-server-mode' else: clone_flags = '' # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace snapshot_test') tablet_62344.init_tablet('master', 'snapshot_test', '0') utils.run_vtctl('RebuildShardGraph snapshot_test/0') utils.validate_topology() tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test, self._populate_vt_insert_test) tablet_62344.start_vttablet() tablet_62044.create_db('vt_snapshot_test') tablet_62044.init_tablet('idle', start=True) # small test to make sure the directory validation works snapshot_dir = os.path.join(utils.vtdataroot, 'snapshot') utils.run("rm -rf %s" % snapshot_dir) utils.run("mkdir -p %s" % snapshot_dir) utils.run("chmod -w %s" % snapshot_dir) out, err = utils.run( utils.vtroot + '/bin/vtctl --alsologtostderr Clone -force %s %s %s' % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias), trap_output=True, raise_on_error=False) if "Cannot validate snapshot directory" not in err: raise utils.TestError("expected validation error", err) if "Un-reserved test_nj-0000062044" not in err: raise utils.TestError("expected Un-reserved", err) logging.debug("Failed Clone output: " + err) utils.run("chmod +w %s" % snapshot_dir) call(["touch", "/tmp/vtSimulateFetchFailures"]) utils.run_vtctl('Clone -force %s %s %s' % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias), auto_log=True) utils.pause("look at logs!") tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) utils.validate_topology() tablet_62344.kill_vttablet() tablet_62044.kill_vttablet()
def test_topocustomrule(self): # Empty rule file. topocustomrule_file = environment.tmproot+'/rules.json' with open(topocustomrule_file, 'w') as fd: fd.write('[]\n') # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace']) utils.run_vtctl(['createshard', '-force', 'test_keyspace/0']) tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) utils.validate_topology() # Copy config file into topo. topocustomrule_path = '/keyspaces/test_keyspace/configs/CustomRules' utils.run_vtctl(['TopoCp', '-to_topo', topocustomrule_file, topocustomrule_path]) # Put some data in, start master. tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet(topocustomrule_path=topocustomrule_path) # make sure the query service is working qr = tablet_62344.execute('select id, msg from vt_select_test') self.assertEqual(len(qr['rows']), 4, 'expected 4 rows in vt_select_test: %s' % str(qr)) # Now update the topocustomrule file. with open(topocustomrule_file, 'w') as fd: fd.write(''' [{ "Name": "rule1", "Description": "disallow select on table vt_select_test", "TableNames" : ["vt_select_test"], "Query" : "(select)|(SELECT)" }]''') utils.run_vtctl(['TopoCp', '-to_topo', topocustomrule_file, topocustomrule_path]) # And wait until the query fails with the right error. timeout = 10.0 while True: try: tablet_62344.execute('select id, msg from vt_select_test') timeout = utils.wait_step('query rule in place', timeout) except Exception as e: print e expected = ('disallowed due to rule: disallow select' ' on table vt_select_test') self.assertIn(expected, str(e)) break # Cleanup. tablet_62344.kill_vttablet()
def test_reparent(self): name = 'sharded_message' keyspace = 'user' # Start a stream. Use a timeout that's greater than how long # the test will take to run. vtgate_conn = get_connection(120) (it, fields) = vtgate_conn.message_stream(keyspace, name) self.assertEqual(get_client_count(shard_0_master), 1) self.assertEqual(get_client_count(shard_0_replica), 0) self.assertEqual(get_client_count(shard_1_master), 1) # Perform a graceful reparent to the replica. utils.run_vtctl([ 'PlannedReparentShard', '-keyspace_shard', 'user/-80', '-new_master', shard_0_replica.tablet_alias ], auto_log=True) utils.validate_topology() # Verify connection has migrated. # The wait must be at least 6s which is how long vtgate will # wait before retrying: that is 30s/5 where 30s is the default # message_stream_grace_period. time.sleep(10) self.assertEqual(get_client_count(shard_0_master), 0) self.assertEqual(get_client_count(shard_0_replica), 1) self.assertEqual(get_client_count(shard_1_master), 1) cursor = vtgate_conn.cursor(tablet_type='master', keyspace=None, writable=True) query = 'insert into %s(id, message) values(:id, :message)' % (name) cursor.begin() cursor.execute(query, {'id': 3, 'message': 'hello world 3'}) cursor.commit() # Receive the message. it.next() # Reparent back to old master. utils.run_vtctl([ 'PlannedReparentShard', '-keyspace_shard', 'user/-80', '-new_master', shard_0_master.tablet_alias ], auto_log=True) utils.validate_topology() time.sleep(10) self.assertEqual(get_client_count(shard_0_master), 1) self.assertEqual(get_client_count(shard_0_replica), 0) self.assertEqual(get_client_count(shard_1_master), 1) # Ack the message. count = vtgate_conn.message_ack(name, [3])
def run_test_scrap(): # Start up a master mysql and vttablet utils.run_vtctl("CreateKeyspace -force test_keyspace") tablet_62344.init_tablet("master", "test_keyspace", "0") tablet_62044.init_tablet("replica", "test_keyspace", "0") utils.run_vtctl("RebuildShardGraph test_keyspace/0") utils.validate_topology() tablet_62044.scrap(force=True) utils.validate_topology()
def setUpModule(): global master_start_position try: environment.topo_server().setup() # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()] utils.wait_procs(setup_procs) # start a vtctld so the vtctl insert commands are just RPCs, not forks utils.Vtctld().start() # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0') replica_tablet.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0']) utils.validate_topology() master_tablet.create_db('vt_test_keyspace') master_tablet.create_db('other_database') replica_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('other_database') utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace']) utils.VtGate().start() master_tablet.start_vttablet() replica_tablet.start_vttablet() utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias]) utils.check_db_read_write(master_tablet.tablet_uid) for t in [master_tablet, replica_tablet]: t.reset_replication() utils.run_vtctl(['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias], auto_log=True) # reset counter so tests don't assert tablet.Tablet.tablets_running = 0 master_start_position = _get_master_current_position() master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test) master_tablet.mquery('vt_test_keyspace', _create_vt_a) master_tablet.mquery('vt_test_keyspace', _create_vt_b) utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias]) utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) except: tearDownModule() raise
def run_test_scrap(): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') tablet_62044.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62044.scrap(force=True) utils.validate_topology()
def test_scrap(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') tablet_62044.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/*']) utils.validate_topology() tablet_62044.scrap(force=True) utils.validate_topology()
def _test_mysqlctl_clone(server_mode): if server_mode: snapshot_cmd = ['snapshotsourcestart', '-concurrency=8'] restore_flags = ['-dont_wait_for_slave_start'] else: snapshot_cmd = ['snapshot', '-concurrency=5'] restore_flags = [] # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'snapshot_test']) tablet_62344.init_tablet('master', 'snapshot_test', '0') utils.run_vtctl(['RebuildShardGraph', 'snapshot_test/0']) utils.validate_topology() tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test, self._populate_vt_insert_test) tablet_62344.start_vttablet() err = tablet_62344.mysqlctl(snapshot_cmd + ['vt_snapshot_test'], with_ports=True).wait() if err != 0: self.fail('mysqlctl %s failed' % str(snapshot_cmd)) utils.pause("%s finished" % str(snapshot_cmd)) call(["touch", "/tmp/vtSimulateFetchFailures"]) err = tablet_31981.mysqlctl( ['restore', '-fetch_concurrency=2', '-fetch_retry_count=4'] + restore_flags + [ environment.vtdataroot + '/snapshot/vt_0000062344/snapshot_manifest.json' ], with_ports=True).wait() if err != 0: self.fail('mysqlctl restore failed') tablet_31981.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) if server_mode: err = tablet_62344.mysqlctl( ['snapshotsourceend', '-read_write', 'vt_snapshot_test'], with_ports=True).wait() if err != 0: self.fail('mysqlctl snapshotsourceend failed') # see if server restarted properly tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) tablet_62344.kill_vttablet()
def _run_test_mysqlctl_clone(server_mode): if server_mode: snapshot_cmd = "snapshotsourcestart -concurrency=8" restore_flags = "-dont-wait-for-slave-start" else: snapshot_cmd = "snapshot -concurrency=5" restore_flags = "" utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force snapshot_test') tablet_62344.init_tablet('master', 'snapshot_test', '0') utils.run_vtctl('RebuildShardGraph snapshot_test/0') utils.validate_topology() tablet_62344.populate('vt_snapshot_test', create_vt_insert_test, populate_vt_insert_test) tablet_62344.start_vttablet() err = tablet_62344.mysqlctl( '-port %u -mysql-port %u %s vt_snapshot_test' % (tablet_62344.port, tablet_62344.mysql_port, snapshot_cmd)).wait() if err != 0: raise utils.TestError('mysqlctl %s failed' % snapshot_cmd) utils.pause("%s finished" % snapshot_cmd) call(["touch", "/tmp/vtSimulateFetchFailures"]) err = tablet_62044.mysqlctl( '-port %u -mysql-port %u restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s/snapshot/vt_0000062344/snapshot_manifest.json' % (tablet_62044.port, tablet_62044.mysql_port, restore_flags, utils.vtdataroot)).wait() if err != 0: raise utils.TestError('mysqlctl restore failed') tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) if server_mode: err = tablet_62344.mysqlctl( '-port %u -mysql-port %u snapshotsourceend -read-write vt_snapshot_test' % (tablet_62344.port, tablet_62344.mysql_port)).wait() if err != 0: raise utils.TestError('mysqlctl snapshotsourceend failed') # see if server restarted properly tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) tablet_62344.kill_vttablet()
def setUpModule(): try: environment.topo_server().setup() setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()] utils.wait_procs(setup_procs) # start a vtctld so the vtctl insert commands are just RPCs, not forks. utils.Vtctld().start() # Start up a master mysql and vttablet logging.debug('Setting up tablets') utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) master_tablet.init_tablet('master', 'test_keyspace', '0', tablet_index=0) replica_tablet.init_tablet('replica', 'test_keyspace', '0', tablet_index=1) utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True) utils.validate_topology() master_tablet.create_db('vt_test_keyspace') replica_tablet.create_db('vt_test_keyspace') master_tablet.start_vttablet(wait_for_state=None) replica_tablet.start_vttablet(wait_for_state=None) master_tablet.wait_for_vttablet_state('SERVING') replica_tablet.wait_for_vttablet_state('NOT_SERVING') utils.run_vtctl( ['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias], auto_log=True) utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica') master_tablet.wait_for_vttablet_state('SERVING') replica_tablet.wait_for_vttablet_state('SERVING') master_tablet.mquery('vt_test_keyspace', _create_vt_a) master_tablet.mquery('vt_test_keyspace', _create_vt_b) utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias]) utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias]) utils.run_vtctl(['RebuildVSchemaGraph']) utils.VtGate().start(tablets=[master_tablet, replica_tablet]) utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1) utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1) except: tearDownModule() raise
def test_restart_during_action(self): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl('Ping ' + tablet_62344.tablet_alias) # schedule long action utils.run_vtctl('-no-wait Sleep %s 15s' % tablet_62344.tablet_alias, stdout=utils.devnull) # ping blocks until the sleep finishes unless we have a schedule race action_path, _ = utils.run_vtctl('-no-wait Ping ' + tablet_62344.tablet_alias, trap_output=True) # kill agent leaving vtaction running tablet_62344.kill_vttablet() # restart agent tablet_62344.start_vttablet() # we expect this action with a short wait time to fail. this isn't the best # and has some potential for flakiness. utils.run_vtctl('-wait-time 2s WaitForAction ' + action_path, expect_fail=True) # wait until the background sleep action is done, otherwise there will be # a leftover vtaction whose result may overwrite running actions # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug: # the zombie Sleep clobbers the Clone command in the following tests utils.run_vtctl('-wait-time 20s WaitForAction ' + action_path, auto_log=True) # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) if v['Voltron']['States']['DurationSERVING'] < 10e9: raise utils.TestError('not enough time in Open state', v['Voltron']['States']['DurationSERVING']) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': raise utils.TestError('invalid zk test_nj state: ', v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': raise utils.TestError('invalid zk global state: ', v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: raise utils.TestError('not enough time in Connected state', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': raise utils.TestError('TabletType not exported correctly') tablet_62344.kill_vttablet()
def test_restart_during_action(self): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl('Ping ' + tablet_62344.tablet_alias) # schedule long action utils.run_vtctl('-no-wait Sleep %s 15s' % tablet_62344.tablet_alias, stdout=utils.devnull) # ping blocks until the sleep finishes unless we have a schedule race action_path, _ = utils.run_vtctl('-no-wait Ping ' + tablet_62344.tablet_alias, trap_output=True) # kill agent leaving vtaction running tablet_62344.kill_vttablet() # restart agent tablet_62344.start_vttablet() # we expect this action with a short wait time to fail. this isn't the best # and has some potential for flakiness. utils.run_fail(utils.vtroot+'/bin/vtctl -log_dir '+utils.tmp_root+' --alsologtostderr -wait-time 2s WaitForAction ' + action_path) # wait until the background sleep action is done, otherwise there will be # a leftover vtaction whose result may overwrite running actions # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug: # the zombie Sleep clobbers the Clone command in the following tests utils.run_vtctl('-wait-time 20s WaitForAction ' + action_path, auto_log=True) # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) logging.debug("vars: %s" % str(v)) if v['Voltron']['States']['DurationSERVING'] < 10e9: raise utils.TestError('not enough time in Open state', v['Voltron']['States']['DurationSERVING']) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': raise utils.TestError('invalid zk test_nj state: ', v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': raise utils.TestError('invalid zk global state: ', v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: raise utils.TestError('not enough time in Connected state', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['TabletType'] != 'master': raise utils.TestError('TabletType not exported correctly') tablet_62344.kill_vttablet()
def _test_vtctl_clone(self, server_mode): if server_mode: clone_flags = ['-server-mode'] else: clone_flags = [] # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'snapshot_test']) tablet_62344.init_tablet('master', 'snapshot_test', '0') utils.run_vtctl(['RebuildShardGraph', 'snapshot_test/0']) utils.validate_topology() tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test, self._populate_vt_insert_test) tablet_62344.start_vttablet() tablet_31981.create_db('vt_snapshot_test') tablet_31981.init_tablet('idle', start=True) # small test to make sure the directory validation works snapshot_dir = os.path.join(environment.vtdataroot, 'snapshot') utils.run("rm -rf %s" % snapshot_dir) utils.run("mkdir -p %s" % snapshot_dir) utils.run("chmod -w %s" % snapshot_dir) out, err = utils.run_vtctl( ['Clone', '-force'] + clone_flags + [tablet_62344.tablet_alias, tablet_31981.tablet_alias], log_level='INFO', expect_fail=True) if "Cannot validate snapshot directory" not in err: self.fail("expected validation error: %s" % err) if "Un-reserved test_ny-0000031981" not in err: self.fail("expected Un-reserved: %s" % err) logging.debug("Failed Clone output: " + err) utils.run("chmod +w %s" % snapshot_dir) call(["touch", "/tmp/vtSimulateFetchFailures"]) utils.run_vtctl(['Clone', '-force'] + clone_flags + [tablet_62344.tablet_alias, tablet_31981.tablet_alias], auto_log=True) self._check_shard() utils.pause("look at logs!") tablet_31981.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) utils.validate_topology() tablet.kill_tablets([tablet_62344, tablet_31981])
def _run_test_vtctl_clone(server_mode): if server_mode: clone_flags = "-server-mode" else: clone_flags = "" utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl("CreateKeyspace -force snapshot_test") tablet_62344.init_tablet("master", "snapshot_test", "0") utils.run_vtctl("RebuildShardGraph snapshot_test/0") utils.validate_topology() tablet_62344.populate("vt_snapshot_test", create_vt_insert_test, populate_vt_insert_test) tablet_62344.start_vttablet() tablet_62044.create_db("vt_snapshot_test") tablet_62044.init_tablet("idle", start=True) # small test to make sure the directory validation works snapshot_dir = os.path.join(utils.vtdataroot, "snapshot") utils.run("rm -rf %s" % snapshot_dir) utils.run("mkdir -p %s" % snapshot_dir) utils.run("chmod -w %s" % snapshot_dir) out, err = utils.run( utils.vtroot + "/bin/vtctl --alsologtostderr Clone -force %s %s %s" % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias), trap_output=True, raise_on_error=False, ) if "Cannot validate snapshot directory" not in err: raise utils.TestError("expected validation error", err) if "Un-reserved test_nj-0000062044" not in err: raise utils.TestError("expected Un-reserved", err) utils.debug("Failed Clone output: " + err) utils.run("chmod +w %s" % snapshot_dir) call(["touch", "/tmp/vtSimulateFetchFailures"]) utils.run_vtctl( "Clone -force %s %s %s" % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias), auto_log=True ) utils.pause("look at logs!") tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4) tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4) utils.validate_topology() tablet_62344.kill_vttablet() tablet_62044.kill_vttablet()
def test_scrap(self): # Start up a master mysql and vttablet utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) tablet_62344.init_tablet('master', 'test_keyspace', '0') tablet_62044.init_tablet('replica', 'test_keyspace', '0') utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/*']) utils.validate_topology() self._check_srv_shard() tablet_62044.scrap(force=True) utils.validate_topology() self._check_srv_shard()
def _run_test_mysqlctl_clone(server_mode): if server_mode: snapshot_cmd = "snapshotsourcestart -concurrency=8" restore_flags = "-dont-wait-for-slave-start" else: snapshot_cmd = "snapshot -concurrency=5" restore_flags = "" utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl("CreateKeyspace -force snapshot_test") tablet_62344.init_tablet("master", "snapshot_test", "0") utils.run_vtctl("RebuildShardGraph snapshot_test/0") utils.validate_topology() tablet_62344.populate("vt_snapshot_test", create_vt_insert_test, populate_vt_insert_test) tablet_62344.start_vttablet() err = tablet_62344.mysqlctl( "-port %u -mysql-port %u %s vt_snapshot_test" % (tablet_62344.port, tablet_62344.mysql_port, snapshot_cmd) ).wait() if err != 0: raise utils.TestError("mysqlctl %s failed" % snapshot_cmd) utils.pause("%s finished" % snapshot_cmd) call(["touch", "/tmp/vtSimulateFetchFailures"]) err = tablet_62044.mysqlctl( "-port %u -mysql-port %u restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s/snapshot/vt_0000062344/snapshot_manifest.json" % (tablet_62044.port, tablet_62044.mysql_port, restore_flags, utils.vtdataroot) ).wait() if err != 0: raise utils.TestError("mysqlctl restore failed") tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4) if server_mode: err = tablet_62344.mysqlctl( "-port %u -mysql-port %u snapshotsourceend -read-write vt_snapshot_test" % (tablet_62344.port, tablet_62344.mysql_port) ).wait() if err != 0: raise utils.TestError("mysqlctl snapshotsourceend failed") # see if server restarted properly tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4) tablet_62344.kill_vttablet()
def run_test_restart_during_action(): # Start up a master mysql and vttablet utils.run_vtctl("CreateKeyspace -force test_keyspace") tablet_62344.init_tablet("master", "test_keyspace", "0") utils.run_vtctl("RebuildShardGraph test_keyspace/0") utils.validate_topology() tablet_62344.create_db("vt_test_keyspace") tablet_62344.start_vttablet() utils.run_vtctl("Ping " + tablet_62344.tablet_alias) # schedule long action utils.run_vtctl("-no-wait Sleep %s 15s" % tablet_62344.tablet_alias, stdout=devnull) # ping blocks until the sleep finishes unless we have a schedule race action_path, _ = utils.run_vtctl("-no-wait Ping " + tablet_62344.tablet_alias, trap_output=True) # kill agent leaving vtaction running tablet_62344.kill_vttablet() # restart agent tablet_62344.start_vttablet() # we expect this action with a short wait time to fail. this isn't the best # and has some potential for flakiness. utils.run_fail(utils.vtroot + "/bin/vtctl --alsologtostderr -wait-time 2s WaitForAction " + action_path) # wait until the background sleep action is done, otherwise there will be # a leftover vtaction whose result may overwrite running actions # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug: # the zombie Sleep clobbers the Clone command in the following tests utils.run(utils.vtroot + "/bin/vtctl --alsologtostderr -wait-time 20s WaitForAction " + action_path) # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) utils.debug("vars: %s" % str(v)) if v["Voltron"]["States"]["DurationOPEN"] < 10e9: raise utils.TestError("not enough time in Open state", v["Voltron"]["States"]["DurationOPEN"]) # then the Zookeeper connections if v["ZkMetaConn"]["test_nj"]["Current"] != "Connected": raise utils.TestError("invalid zk test_nj state: ", v["ZkMetaConn"]["test_nj"]["Current"]) if v["ZkMetaConn"]["global"]["Current"] != "Connected": raise utils.TestError("invalid zk global state: ", v["ZkMetaConn"]["global"]["Current"]) if v["ZkMetaConn"]["test_nj"]["DurationConnected"] < 10e9: raise utils.TestError("not enough time in Connected state", v["ZkMetaConn"]["test_nj"]["DurationConnected"]) if v["tablet-type"] != "master": raise utils.TestError("tablet-type not exported correctly") tablet_62344.kill_vttablet()
def test_multisnapshot_and_restore_vtctl(): tables = ['vt_insert_test', 'vt_insert_test1'] create_template = '''create table %s ( id bigint auto_increment, msg varchar(64), primary key (id) ) Engine=InnoDB''' insert_template = "insert into %s (id, msg) values (%s, 'test %s')" utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') # Start three tablets for three different shards. At this point the # sharding schema is not really important, as long as it is # consistent. new_spec = '-0000000000000028-' old_tablets = [tablet_62044, tablet_41983, tablet_31981] for i, tablet in enumerate(old_tablets): tablet.init_tablet('master', 'test_keyspace', str(i)) utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i) utils.validate_topology() for i, tablet in enumerate(old_tablets): tablet.populate( "vt_test_keyspace", [create_template % table for table in tables], sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], [])) tablet.start_vttablet() utils.run_vtctl('MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True) utils.run_vtctl('CreateKeyspace -force test_keyspace_new') tablet_62344.init_tablet('master', 'test_keyspace_new', '-0000000000000028', dbname='not_vt_test_keyspace') utils.run_vtctl('RebuildShardGraph test_keyspace_new/-0000000000000028') utils.validate_topology() tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace') tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created # 0x28 = 40 source_aliases = ' '.join(t.tablet_alias for t in old_tablets) utils.run_vtctl('MultiRestore %s %s' % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True) time.sleep(1) for table in tables: rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table) if len(rows) == 0: raise utils.TestError("There are no rows in the restored database.") for row in rows: if row[0] > 32: raise utils.TestError("Bad row: %s" % row) for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344: tablet.kill_vttablet()
def test_reparent_doesnt_hang_if_master_fails(self): """Makes sure a failed master populate doesn't hang.""" utils.run_vtctl(['CreateKeyspace', 'test_keyspace']) # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up vttablet for t in [tablet_62344, tablet_62044, tablet_31981, tablet_41983]: t.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False) # wait for all tablets to start for t in [tablet_62344, tablet_62044, tablet_31981, tablet_41983]: t.wait_for_vttablet_state('NOT_SERVING') # Force the slaves to reparent. Will create the _vt.reparent_journal table. utils.run_vtctl([ 'InitShardMaster', '-force', 'test_keyspace/0', tablet_62344.tablet_alias ]) utils.validate_topology(ping_tablets=True) # Change the schema of the _vt.reparent_journal table, so that # inserts into it will fail. That will make the master fail. tablet_62344.mquery( '_vt', 'ALTER TABLE reparent_journal' ' DROP COLUMN replication_position') # Perform a planned reparent operation, the master will fail the # insert. The slaves should then abort right away. If this fails, # the test will timeout. _, stderr = utils.run_vtctl([ '-wait-time', '3600s', 'PlannedReparentShard', '-keyspace_shard', 'test_keyspace/0', '-new_master', tablet_62044.tablet_alias ], expect_fail=True) self.assertIn( 'master failed to PopulateReparentJournal, canceling slaves', stderr) # Clean up the tablets. tablet.kill_tablets( [tablet_62344, tablet_62044, tablet_41983, tablet_31981])