def setup_tablets():
    # Start up a master mysql and vttablet
    utils.debug("Setting up tablets")
    utils.run_vtctl("CreateKeyspace test_keyspace")
    master_tablet.init_tablet("master", "test_keyspace", "0")
    utils.run_vtctl("RebuildShardGraph test_keyspace/0")
    utils.run_vtctl("RebuildKeyspaceGraph test_keyspace")
    utils.validate_topology()

    setup_schema()
    replica_tablet.create_db("vt_test_keyspace")
    # master_tablet.start_vttablet(auth=True)
    master_tablet.start_vttablet()

    replica_tablet.init_tablet("idle", "test_keyspace", start=True)
    snapshot_dir = os.path.join(utils.vtdataroot, "snapshot")
    utils.run("mkdir -p " + snapshot_dir)
    utils.run("chmod +w " + snapshot_dir)
    utils.run_vtctl("Clone -force %s %s" % (master_tablet.tablet_alias, replica_tablet.tablet_alias))

    utils.run_vtctl("Ping test_nj-0000062344")
    utils.run_vtctl("SetReadWrite " + master_tablet.tablet_alias)
    utils.check_db_read_write(62344)

    utils.validate_topology()
    utils.run_vtctl("Ping test_nj-0000062345")

    # reset counter so tests don't assert
    tablet.Tablet.tablets_running = 0
Beispiel #2
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.validate_topology()
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server().flags() +
            ['-tablet_manager_protocol',
             protocols_flavor().tablet_manager_protocol(),
             '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
             '-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RefreshState that should timeout as the tablet is busy
    # running the other one
    _, stderr = utils.run_vtctl(
        ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
        expect_fail=True)
    self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

    # wait for the background vtctl
    bg.wait()

    tablet_62344.kill_vttablet()
Beispiel #3
0
    def test_restart_during_action(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        tablet_62344.init_tablet('master', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
        utils.validate_topology()
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62344.start_vttablet()

        utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

        # schedule long action
        utils.run_vtctl(
            ['-no-wait', 'Sleep', tablet_62344.tablet_alias, '15s'],
            stdout=utils.devnull)
        # ping blocks until the sleep finishes unless we have a schedule race
        action_path, _ = utils.run_vtctl(
            ['-no-wait', 'Ping', tablet_62344.tablet_alias], trap_output=True)
        action_path = action_path.strip()

        # kill agent leaving vtaction running
        tablet_62344.kill_vttablet()

        # restart agent
        tablet_62344.start_vttablet()

        # we expect this action with a short wait time to fail. this isn't the best
        # and has some potential for flakiness.
        utils.run_vtctl(['-wait-time', '2s', 'WaitForAction', action_path],
                        expect_fail=True)

        # wait until the background sleep action is done, otherwise there will be
        # a leftover vtaction whose result may overwrite running actions
        # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug:
        # the zombie Sleep clobbers the Clone command in the following tests
        utils.run_vtctl(['-wait-time', '20s', 'WaitForAction', action_path],
                        auto_log=True)

        if environment.topo_server_implementation == 'zookeeper':
            # extra small test: we ran for a while, get the states we were in,
            # make sure they're accounted for properly
            # first the query engine States
            v = utils.get_vars(tablet_62344.port)
            logging.debug("vars: %s" % str(v))

            # then the Zookeeper connections
            if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
                self.fail('invalid zk test_nj state: %s' %
                          v['ZkMetaConn']['test_nj']['Current'])
            if v['ZkMetaConn']['global']['Current'] != 'Connected':
                self.fail('invalid zk global state: %s' %
                          v['ZkMetaConn']['global']['Current'])
            if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
                self.fail('not enough time in Connected state: %u',
                          v['ZkMetaConn']['test_nj']['DurationConnected'])
            if v['TabletType'] != 'master':
                self.fail('TabletType not exported correctly')

        tablet_62344.kill_vttablet()
def setup_tablets():
  # Start up a master mysql and vttablet
  utils.debug("Setting up tablets")
  utils.run_vtctl('CreateKeyspace test_keyspace')
  master_tablet.init_tablet('master', 'test_keyspace', '0')
  utils.run_vtctl('RebuildShardGraph test_keyspace/0')
  utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
  utils.validate_topology()

  setup_schema()
  replica_tablet.create_db('vt_test_keyspace')
  master_tablet.start_vttablet(memcache=True)

  replica_tablet.init_tablet('idle', 'test_keyspace', start=True, memcache=True)
  snapshot_dir = os.path.join(utils.vtdataroot, 'snapshot')
  utils.run("mkdir -p " + snapshot_dir)
  utils.run("chmod +w " + snapshot_dir)
  utils.run_vtctl('Clone -force %s %s' %
                  (master_tablet.tablet_alias, replica_tablet.tablet_alias))

  utils.run_vtctl('Ping test_nj-0000062344')
  utils.run_vtctl('SetReadWrite ' + master_tablet.tablet_alias)
  utils.check_db_read_write(62344)

  utils.validate_topology()
  utils.run_vtctl('Ping test_nj-0000062345')
  utils.run_vtctl('ChangeSlaveType test_nj-0000062345 replica')
Beispiel #5
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.validate_topology()
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server().flags() +
            ['-tablet_manager_protocol',
             protocols_flavor().tablet_manager_protocol(),
             '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
             '-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RefreshState that should timeout as the tablet is busy
    # running the other one
    _, stderr = utils.run_vtctl(
        ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
        expect_fail=True)
    self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

    # wait for the background vtctl
    bg.wait()

    tablet_62344.kill_vttablet()
Beispiel #6
0
  def _test_sanity(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace -force test_keyspace')
    utils.run_vtctl('CreateShard -force test_keyspace/0')
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
    utils.validate_topology()

    # if these statements don't run before the tablet it will wedge waiting for the
    # db to become accessible. this is more a bug than a feature.
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()

    # make sure the query service is started right away
    result, _ = utils.run_vtctl('Query test_nj test_keyspace "select * from vt_select_test"', trap_output=True)
    rows = result.splitlines()
    self.assertEqual(len(rows), 5, "expected 5 rows in vt_select_test: %s %s" % (str(rows), result))

    # check Pings
    utils.run_vtctl('Ping ' + tablet_62344.tablet_alias)
    utils.run_vtctl('RpcPing ' + tablet_62344.tablet_alias)

    utils.validate_topology()
    utils.run_vtctl('ValidateKeyspace test_keyspace')
    # not pinging tablets, as it enables replication checks, and they
    # break because we only have a single master, no slaves
    utils.run_vtctl('ValidateShard -ping-tablets=false test_keyspace/0')
Beispiel #7
0
def setUpModule():
    try:
        environment.topo_server().setup()

        _init_mysql(tablets)

        utils.run_vtctl(['CreateKeyspace', test_keyspace])

        shard_0_master.init_tablet('master', test_keyspace, '0')
        shard_0_replica1.init_tablet('replica', test_keyspace, '0')
        shard_0_replica2.init_tablet('replica', test_keyspace, '0')
        shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0')
        shard_0_backup.init_tablet('backup', test_keyspace, '0')
        shard_1_master.init_tablet('master', test_keyspace, '1')
        shard_1_replica1.init_tablet('replica', test_keyspace, '1')

        # run checks now before we start the tablets
        utils.validate_topology()

        utils.Vtctld().start()

        # create databases, start the tablets
        for t in tablets:
            t.create_db(db_name)
            t.start_vttablet(wait_for_state=None)

        # wait for the tablets to start
        shard_0_master.wait_for_vttablet_state('SERVING')
        shard_0_replica1.wait_for_vttablet_state('SERVING')
        shard_0_replica2.wait_for_vttablet_state('SERVING')
        shard_0_rdonly.wait_for_vttablet_state('SERVING')
        shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
        shard_1_master.wait_for_vttablet_state('SERVING')
        shard_1_replica1.wait_for_vttablet_state('SERVING')

        # make sure all replication is good
        for t in tablets:
            t.reset_replication()

        utils.run_vtctl([
            'InitShardMaster', test_keyspace + '/0',
            shard_0_master.tablet_alias
        ],
                        auto_log=True)
        utils.run_vtctl([
            'InitShardMaster', test_keyspace + '/1',
            shard_1_master.tablet_alias
        ],
                        auto_log=True)
        utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace])

        # check after all tablets are here and replication is fixed
        utils.validate_topology(ping_tablets=True)
    except Exception as setup_exception:
        try:
            tearDownModule()
        except Exception as e:
            logging.exception('Tearing down a failed setUpModule() failed: %s',
                              e)
        raise setup_exception
Beispiel #8
0
def setup_tablets():
    # Start up a master mysql and vttablet
    utils.debug("Setting up tablets")
    utils.run_vtctl('CreateKeyspace test_keyspace')
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
    utils.validate_topology()

    setup_schema()
    replica_tablet.create_db('vt_test_keyspace')
    master_tablet.start_vttablet(memcache=True)

    replica_tablet.init_tablet('idle',
                               'test_keyspace',
                               start=True,
                               memcache=True)
    snapshot_dir = os.path.join(utils.vtdataroot, 'snapshot')
    utils.run("mkdir -p " + snapshot_dir)
    utils.run("chmod +w " + snapshot_dir)
    utils.run_vtctl('Clone -force %s %s' %
                    (master_tablet.tablet_alias, replica_tablet.tablet_alias))

    utils.run_vtctl('Ping test_nj-0000062344')
    utils.run_vtctl('SetReadWrite ' + master_tablet.tablet_alias)
    utils.check_db_read_write(62344)

    utils.validate_topology()
    utils.run_vtctl('Ping test_nj-0000062345')
    utils.run_vtctl('ChangeSlaveType test_nj-0000062345 replica')
Beispiel #9
0
    def _test_sanity(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl('CreateKeyspace -force test_keyspace')
        utils.run_vtctl('CreateShard -force test_keyspace/0')
        tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
        utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
        utils.validate_topology()

        # if these statements don't run before the tablet it will wedge waiting for the
        # db to become accessible. this is more a bug than a feature.
        tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                              self._populate_vt_select_test)

        tablet_62344.start_vttablet()

        # make sure the query service is started right away
        result, _ = utils.run_vtctl(
            'Query test_nj test_keyspace "select * from vt_select_test"',
            trap_output=True)
        rows = result.splitlines()
        self.assertEqual(
            len(rows), 5,
            "expected 5 rows in vt_select_test: %s %s" % (str(rows), result))

        # check Pings
        utils.run_vtctl('Ping ' + tablet_62344.tablet_alias)
        utils.run_vtctl('RpcPing ' + tablet_62344.tablet_alias)

        utils.validate_topology()
        utils.run_vtctl('ValidateKeyspace test_keyspace')
        # not pinging tablets, as it enables replication checks, and they
        # break because we only have a single master, no slaves
        utils.run_vtctl('ValidateShard -ping-tablets=false test_keyspace/0')
Beispiel #10
0
def setup_tablets():
  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl('CreateKeyspace test_keyspace')
  master_tablet.init_tablet('master', 'test_keyspace', '0')
  replica_tablet.init_tablet('replica', 'test_keyspace', '0')
  utils.run_vtctl('RebuildShardGraph test_keyspace/0')
  utils.validate_topology()
  master_tablet.create_db('vt_test_keyspace')
  replica_tablet.create_db('vt_test_keyspace')

  utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
  zkocc_server = utils.zkocc_start()

  master_tablet.start_vttablet()
  replica_tablet.start_vttablet()
  utils.run_vtctl('SetReadWrite ' + master_tablet.tablet_alias)
  utils.check_db_read_write(62344)

  for t in [master_tablet, replica_tablet]:
    t.reset_replication()
  utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + master_tablet.tablet_alias, auto_log=True)

  # reset counter so tests don't assert
  tablet.Tablet.tablets_running = 0
  setup_schema()
  master_tablet.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
  replica_tablet.vquery("set vt_schema_reload_time=86400", path="test_keyspace/0")
Beispiel #11
0
  def _test_sanity(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace'])
    utils.run_vtctl(['createshard', '-force', 'test_keyspace/0'])
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
    utils.validate_topology()
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')

    # if these statements don't run before the tablet it will wedge
    # waiting for the db to become accessible. this is more a bug than
    # a feature.
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()

    # make sure the query service is started right away
    result, _ = utils.run_vtctl(['Query', 'test_nj', 'test_keyspace',
                                 'select * from vt_select_test'],
                                mode=utils.VTCTL_VTCTL, trap_output=True)
    rows = result.splitlines()
    self.assertEqual(len(rows), 5, "expected 5 rows in vt_select_test: %s %s" %
                     (str(rows), result))

    # make sure direct dba queries work
    query_result = utils.run_vtctl_json(['ExecuteFetch', '-want_fields', tablet_62344.tablet_alias, 'select * from vt_test_keyspace.vt_select_test'])
    self.assertEqual(len(query_result['Rows']), 4, "expected 4 rows in vt_select_test: %s" % str(query_result))
    self.assertEqual(len(query_result['Fields']), 2, "expected 2 fields in vt_select_test: %s" % str(query_result))

    # check Pings
    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
    utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias])

    # Quickly check basic actions.
    utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias])
    utils.wait_db_read_only(62344)

    utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias])
    utils.check_db_read_write(62344)

    utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias])
    utils.wait_db_read_only(62344)

    utils.validate_topology()
    utils.run_vtctl(['ValidateKeyspace', 'test_keyspace'])
    # not pinging tablets, as it enables replication checks, and they
    # break because we only have a single master, no slaves
    utils.run_vtctl(['ValidateShard', '-ping-tablets=false',
                     'test_keyspace/0'])
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')

    tablet_62344.kill_vttablet()

    tablet_62344.init_tablet('idle')
    tablet_62344.scrap(force=True)
Beispiel #12
0
    def setUpClass(klass):
        utils.run_vtctl("CreateKeyspace test_keyspace")

        shard_0_master.init_tablet("master", "test_keyspace", "-80")
        shard_0_replica.init_tablet("replica", "test_keyspace", "-80")
        shard_0_spare.init_tablet("spare", "test_keyspace", "-80")
        shard_1_master.init_tablet("master", "test_keyspace", "80-")
        shard_1_replica.init_tablet("replica", "test_keyspace", "80-")
        idle.init_tablet("idle")
        scrap.init_tablet("idle")

        utils.run_vtctl("RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*", auto_log=True)
        utils.run_vtctl("RebuildKeyspaceGraph /zk/global/vt/keyspaces/*", auto_log=True)

        for t in assigned:
            t.create_db("vt_test_keyspace")
            t.start_vttablet()

        for t in scrap, idle, shard_0_spare:
            t.start_vttablet(wait_for_state="NOT_SERVING")

        scrap.scrap()

        for t in [shard_0_master, shard_0_replica, shard_0_spare, shard_1_master, shard_1_replica, idle, scrap]:
            t.reset_replication()
        utils.run_vtctl("ReparentShard -force test_keyspace/-80 " + shard_0_master.tablet_alias, auto_log=True)
        utils.run_vtctl("ReparentShard -force test_keyspace/80- " + shard_1_master.tablet_alias, auto_log=True)

        # run checks now before we start the tablets
        utils.validate_topology()
Beispiel #13
0
    def test_actions_and_timeouts(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        tablet_62344.init_tablet('master', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
        utils.validate_topology()
        srvShard = utils.run_vtctl_json(
            ['GetSrvShard', 'test_nj', 'test_keyspace/0'])
        self.assertEqual(srvShard['MasterCell'], 'test_nj')
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62344.start_vttablet()

        utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

        # schedule long action in the background, sleep a little bit to make sure
        # it started to run
        args = (environment.binary_args('vtctl') +
                environment.topo_server().flags() + [
                    '-tablet_manager_protocol',
                    protocols_flavor().tablet_manager_protocol(),
                    '-tablet_protocol',
                    protocols_flavor().tabletconn_protocol(), '-log_dir',
                    environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias,
                    '10s'
                ])
        bg = utils.run_bg(args)
        time.sleep(3)

        # try a frontend RefreshState that should timeout as the tablet is busy
        # running the other one
        stdout, stderr = utils.run_vtctl(
            ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
            expect_fail=True)
        self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

        # wait for the background vtctl
        bg.wait()

        if environment.topo_server().flavor() == 'zookeeper':
            # extra small test: we ran for a while, get the states we were in,
            # make sure they're accounted for properly
            # first the query engine States
            v = utils.get_vars(tablet_62344.port)
            logging.debug("vars: %s" % str(v))

            # then the Zookeeper connections
            if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
                self.fail('invalid zk test_nj state: %s' %
                          v['ZkMetaConn']['test_nj']['Current'])
            if v['ZkMetaConn']['global']['Current'] != 'Connected':
                self.fail('invalid zk global state: %s' %
                          v['ZkMetaConn']['global']['Current'])
            if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
                self.fail('not enough time in Connected state: %u',
                          v['ZkMetaConn']['test_nj']['DurationConnected'])
            if v['TabletType'] != 'master':
                self.fail('TabletType not exported correctly')

        tablet_62344.kill_vttablet()
Beispiel #14
0
  def setUpClass(klass):
    utils.run_vtctl('CreateKeyspace test_keyspace')

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
    shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
    idle.init_tablet('idle')
    scrap.init_tablet('idle')

    utils.run_vtctl('RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*', auto_log=True)
    utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True)

    for t in assigned:
      t.create_db('vt_test_keyspace')
      t.start_vttablet()

    for t in scrap, idle, shard_0_spare:
      t.start_vttablet(wait_for_state='NOT_SERVING')

    scrap.scrap()

    utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' + shard_0_master.tablet_alias, auto_log=True)
    utils.run_vtctl('ReparentShard -force test_keyspace/80- ' + shard_1_master.tablet_alias, auto_log=True)


  # run checks now before we start the tablets
    utils.validate_topology()
Beispiel #15
0
def test_multisnapshot_vtctl():
  populate = sum([[
    "insert into vt_insert_test_%s (msg) values ('test %s')" % (i, x)
    for x in xrange(4)] for i in range(6)], [])
  create = ['''create table vt_insert_test_%s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB''' % i for i in range(6)]

  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  tablet_62344.init_tablet('master', 'test_keyspace', '0')
  utils.run_vtctl('RebuildShardGraph test_keyspace/0')
  utils.validate_topology()

  tablet_62344.populate('vt_test_keyspace', create,
                        populate)

  tablet_62344.start_vttablet()

  utils.run_vtctl('MultiSnapshot --force --tables=vt_insert_test_1,vt_insert_test_2,vt_insert_test_3 --spec=-0000000000000003- %s id' % tablet_62344.tablet_alias)

  # if err != 0:
  #   raise utils.TestError('mysqlctl multisnapshot failed')
  if os.path.exists(os.path.join(utils.vtdataroot, 'snapshot/vt_0000062344/data/vt_test_keyspace-,0000000000000003/vt_insert_test_4.0.csv.gz')):
    raise utils.TestError("Table vt_insert_test_4 wasn't supposed to be dumped.")
  for kr in 'vt_test_keyspace-,0000000000000003', 'vt_test_keyspace-0000000000000003,':
    path = os.path.join(utils.vtdataroot, 'snapshot/vt_0000062344/data/', kr, 'vt_insert_test_1.0.csv.gz')
    with gzip.open(path) as f:
      if len(f.readlines()) != 2:
        raise utils.TestError("Data looks wrong in %s" % path)
Beispiel #16
0
  def _test_sanity(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace'])
    utils.run_vtctl(['createshard', '-force', 'test_keyspace/0'])
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
    utils.validate_topology()
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')

    # if these statements don't run before the tablet it will wedge
    # waiting for the db to become accessible. this is more a bug than
    # a feature.
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()

    # make sure the query service is started right away
    result, _ = utils.run_vtctl(['Query', 'test_nj', 'test_keyspace',
                                 'select * from vt_select_test'],
                                mode=utils.VTCTL_VTCTL, trap_output=True)
    rows = result.splitlines()
    self.assertEqual(len(rows), 5, "expected 5 rows in vt_select_test: %s %s" %
                     (str(rows), result))

    # make sure direct dba queries work
    query_result = utils.run_vtctl_json(['ExecuteFetchAsDba', '-want_fields', tablet_62344.tablet_alias, 'select * from vt_test_keyspace.vt_select_test'])
    self.assertEqual(len(query_result['Rows']), 4, "expected 4 rows in vt_select_test: %s" % str(query_result))
    self.assertEqual(len(query_result['Fields']), 2, "expected 2 fields in vt_select_test: %s" % str(query_result))

    # check Ping / RefreshState
    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
    utils.run_vtctl(['RefreshState', tablet_62344.tablet_alias])

    # Quickly check basic actions.
    utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias])
    utils.wait_db_read_only(62344)

    utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias])
    utils.check_db_read_write(62344)

    utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias])
    utils.wait_db_read_only(62344)

    utils.validate_topology()
    utils.run_vtctl(['ValidateKeyspace', 'test_keyspace'])
    # not pinging tablets, as it enables replication checks, and they
    # break because we only have a single master, no slaves
    utils.run_vtctl(['ValidateShard', '-ping-tablets=false',
                     'test_keyspace/0'])
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')

    tablet_62344.kill_vttablet()

    tablet_62344.init_tablet('idle')
    tablet_62344.scrap(force=True)
Beispiel #17
0
def run_test_reparent_lag_slave(shard_id='0'):
    utils.zk_wipe()

    utils.run_vtctl('CreateKeyspace -force test_keyspace')

    # create the database so vttablets start, as they are serving
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62044.create_db('vt_test_keyspace')
    tablet_41983.create_db('vt_test_keyspace')
    tablet_31981.create_db('vt_test_keyspace')

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True)

    # Create a few slaves for testing reparenting.
    tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True)
    tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True)
    tablet_41983.init_tablet('lag', 'test_keyspace', shard_id, start=True)

    # Recompute the shard layout node - until you do that, it might not be valid.
    utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id)
    utils.validate_topology()

    # Force the slaves to reparent assuming that all the datasets are identical.
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
        t.reset_replication()
    utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' %
                    (shard_id, tablet_62344.tablet_alias))
    utils.validate_topology(ping_tablets=True)

    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.mquery('vt_test_keyspace', create_vt_insert_test)

    tablet_41983.mquery('', 'stop slave')
    for q in populate_vt_insert_test:
        tablet_62344.mquery('vt_test_keyspace', q, write=True)

    # Perform a graceful reparent operation.
    utils.run_vtctl('ReparentShard test_keyspace/%s %s' %
                    (shard_id, tablet_62044.tablet_alias))

    tablet_41983.mquery('', 'start slave')
    time.sleep(1)

    utils.pause("check orphan")

    utils.run_vtctl('ReparentTablet %s' % tablet_41983.tablet_alias)

    result = tablet_41983.mquery('vt_test_keyspace',
                                 'select msg from vt_insert_test where id=1')
    if len(result) != 1:
        raise utils.TestError('expected 1 row from vt_insert_test', result)

    utils.pause("check lag reparent")

    tablet_62344.kill_vttablet()
    tablet_62044.kill_vttablet()
    tablet_41983.kill_vttablet()
    tablet_31981.kill_vttablet()
Beispiel #18
0
    def test_reparent_cross_cell(self, shard_id='0'):
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        # create the database so vttablets start, as they are serving
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62044.create_db('vt_test_keyspace')
        tablet_41983.create_db('vt_test_keyspace')
        tablet_31981.create_db('vt_test_keyspace')

        # Start up a master mysql and vttablet
        tablet_62344.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)

        # Create a few slaves for testing reparenting. Won't be healthy
        # as replication is not running.
        tablet_62044.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_41983.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_31981.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
            t.wait_for_vttablet_state('NOT_SERVING')

        # Force the slaves to reparent assuming that all the datasets are
        # identical.
        utils.run_vtctl([
            'InitShardMaster', '-force', 'test_keyspace/' + shard_id,
            tablet_62344.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology(ping_tablets=True)

        self._check_master_tablet(tablet_62344)

        # Perform a graceful reparent operation to another cell.
        utils.run_vtctl([
            'PlannedReparentShard', '-keyspace_shard', 'test_keyspace/' +
            shard_id, '-new_master', tablet_31981.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology()

        self._check_master_tablet(tablet_31981)

        tablet.kill_tablets(
            [tablet_62344, tablet_62044, tablet_41983, tablet_31981])
Beispiel #19
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['RpcPing', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server_flags() +
            environment.tablet_manager_protocol_flags() +
            environment.tabletconn_protocol_flags() +
            ['-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RpcPing that should timeout as the tablet is busy
    # running the other one
    stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
                                      'RpcPing', tablet_62344.tablet_alias],
                                     expect_fail=True)
    if 'Timeout waiting for' not in stderr:
      self.fail("didn't find the right error strings in failed RpcPing: " +
                stderr)

    # wait for the background vtctl
    bg.wait()

    if environment.topo_server_implementation == 'zookeeper':
      # extra small test: we ran for a while, get the states we were in,
      # make sure they're accounted for properly
      # first the query engine States
      v = utils.get_vars(tablet_62344.port)
      logging.debug("vars: %s" % str(v))

      # then the Zookeeper connections
      if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
        self.fail('invalid zk test_nj state: %s' %
                  v['ZkMetaConn']['test_nj']['Current'])
      if v['ZkMetaConn']['global']['Current'] != 'Connected':
        self.fail('invalid zk global state: %s' %
                  v['ZkMetaConn']['global']['Current'])
      if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
        self.fail('not enough time in Connected state: %u',
                  v['ZkMetaConn']['test_nj']['DurationConnected'])
      if v['TabletType'] != 'master':
        self.fail('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
Beispiel #20
0
    def setUpClass(klass):
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        utils.run_vtctl([
            'CreateKeyspace', '--served-from',
            'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
            'redirected_keyspace'
        ])

        shard_0_master.init_tablet('master', 'test_keyspace', '-80')
        shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
        shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
        shard_1_master.init_tablet('master', 'test_keyspace', '80-')
        shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
        idle.init_tablet('idle')
        scrap.init_tablet('idle')

        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                        auto_log=True)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
                        auto_log=True)

        for t in [shard_0_master, shard_1_master, shard_1_replica]:
            t.create_db('vt_test_keyspace')
            t.start_vttablet(extra_args=vtctld.process_args())
        shard_0_replica.create_db('vt_test_keyspace')
        shard_0_replica.start_vttablet(extra_args=vtctld.process_args(),
                                       target_tablet_type='replica',
                                       wait_for_state='NOT_SERVING')

        for t in scrap, idle, shard_0_spare:
            t.start_vttablet(wait_for_state='NOT_SERVING',
                             extra_args=vtctld.process_args())

        scrap.scrap()

        for t in [
                shard_0_master, shard_0_replica, shard_0_spare, shard_1_master,
                shard_1_replica, idle, scrap
        ]:
            t.reset_replication()
        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/-80',
            shard_0_master.tablet_alias
        ],
                        auto_log=True)
        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/80-',
            shard_1_master.tablet_alias
        ],
                        auto_log=True)
        shard_0_replica.wait_for_vttablet_state('SERVING')

        # run checks now before we start the tablets
        utils.validate_topology()

        # start a vtgate server too
        global vtgate_server, vtgate_port
        vtgate_server, vtgate_port = utils.vtgate_start(
            cache_ttl='0s', extra_args=vtctld.process_args())
Beispiel #21
0
    def _test_reparent_slave_offline(self, shard_id='0'):
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        # create the database so vttablets start, as they are serving
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62044.create_db('vt_test_keyspace')
        tablet_41983.create_db('vt_test_keyspace')
        tablet_31981.create_db('vt_test_keyspace')

        # Start up a master mysql and vttablet
        tablet_62344.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)

        # Create a few slaves for testing reparenting.
        tablet_62044.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_41983.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_31981.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)

        # wait for all tablets to start
        for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
            t.wait_for_vttablet_state('NOT_SERVING')

        # Force the slaves to reparent assuming that all the datasets are
        # identical.
        utils.run_vtctl([
            'InitShardMaster', '-force', 'test_keyspace/' + shard_id,
            tablet_62344.tablet_alias
        ])
        utils.validate_topology(ping_tablets=True)

        self._check_master_tablet(tablet_62344)

        # Kill one tablet so we seem offline
        tablet_31981.kill_vttablet()

        # Perform a graceful reparent operation.
        utils.run_vtctl([
            'PlannedReparentShard', '-keyspace_shard',
            'test_keyspace/' + shard_id, '-new_master',
            tablet_62044.tablet_alias
        ])
        self._check_master_tablet(tablet_62044)

        tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983])
Beispiel #22
0
  def test_restart_during_action(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj', 'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action
    utils.run_vtctl(['-no-wait', 'Sleep', tablet_62344.tablet_alias, '15s'], stdout=utils.devnull)
    # ping blocks until the sleep finishes unless we have a schedule race
    action_path, _ = utils.run_vtctl(['-no-wait', 'Ping', tablet_62344.tablet_alias], trap_output=True)
    action_path = action_path.strip()

    # kill agent leaving vtaction running
    tablet_62344.kill_vttablet()

    # restart agent
    tablet_62344.start_vttablet()

    # we expect this action with a short wait time to fail. this isn't the best
    # and has some potential for flakiness.
    utils.run_vtctl(['-wait-time', '2s', 'WaitForAction', action_path],
                    expect_fail=True)

    # wait until the background sleep action is done, otherwise there will be
    # a leftover vtaction whose result may overwrite running actions
    # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug:
    # the zombie Sleep clobbers the Clone command in the following tests
    utils.run_vtctl(['-wait-time', '20s', 'WaitForAction', action_path],
                    auto_log=True)

    if environment.topo_server_implementation == 'zookeeper':
      # extra small test: we ran for a while, get the states we were in,
      # make sure they're accounted for properly
      # first the query engine States
      v = utils.get_vars(tablet_62344.port)
      logging.debug("vars: %s" % str(v))

      # then the Zookeeper connections
      if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
        self.fail('invalid zk test_nj state: %s' %
                  v['ZkMetaConn']['test_nj']['Current'])
      if v['ZkMetaConn']['global']['Current'] != 'Connected':
        self.fail('invalid zk global state: %s' %
                  v['ZkMetaConn']['global']['Current'])
      if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
        self.fail('not enough time in Connected state: %u',
                  v['ZkMetaConn']['test_nj']['DurationConnected'])
      if v['TabletType'] != 'master':
        self.fail('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
Beispiel #23
0
    def setUpClass(cls):
        utils.run_vtctl([
            'CreateKeyspace', '--sharding_column_name', 'keyspace_id',
            '--sharding_column_type', 'uint64', 'test_keyspace'
        ])
        utils.run_vtctl([
            'CreateKeyspace', '--served_from',
            'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
            'redirected_keyspace'
        ])

        shard_0_master.init_tablet('master', 'test_keyspace', '-80')
        shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
        shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
        shard_1_master.init_tablet('master', 'test_keyspace', '80-')
        shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')

        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                        auto_log=True)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
                        auto_log=True)

        # start running all the tablets
        for t in [shard_0_master, shard_1_master, shard_1_replica]:
            t.create_db('vt_test_keyspace')
            t.start_vttablet(wait_for_state=None,
                             extra_args=utils.vtctld.process_args())
        shard_0_replica.create_db('vt_test_keyspace')
        shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(),
                                       target_tablet_type='replica',
                                       wait_for_state=None)

        shard_0_spare.start_vttablet(wait_for_state=None,
                                     extra_args=utils.vtctld.process_args())

        # wait for the right states
        for t in [shard_0_master, shard_1_master, shard_1_replica]:
            t.wait_for_vttablet_state('SERVING')
        for t in [shard_0_replica, shard_0_spare]:
            t.wait_for_vttablet_state('NOT_SERVING')

        for t in [
                shard_0_master, shard_0_replica, shard_0_spare, shard_1_master,
                shard_1_replica
        ]:
            t.reset_replication()
        utils.run_vtctl([
            'InitShardMaster', 'test_keyspace/-80', shard_0_master.tablet_alias
        ],
                        auto_log=True)
        utils.run_vtctl([
            'InitShardMaster', 'test_keyspace/80-', shard_1_master.tablet_alias
        ],
                        auto_log=True)
        shard_0_replica.wait_for_vttablet_state('SERVING')

        # run checks now
        utils.validate_topology()
Beispiel #24
0
def setUpModule():
    global vtgate_server
    global vtgate_port
    global vtgate_socket_file
    global master_start_position

    try:
        environment.topo_server().setup()

        # start mysql instance external to the test
        setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
        utils.wait_procs(setup_procs)

        # Start up a master mysql and vttablet
        logging.debug('Setting up tablets')
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        master_tablet.init_tablet('master', 'test_keyspace', '0')
        replica_tablet.init_tablet('replica', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
        utils.validate_topology()
        master_tablet.create_db('vt_test_keyspace')
        master_tablet.create_db('other_database')
        replica_tablet.create_db('vt_test_keyspace')
        replica_tablet.create_db('other_database')

        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

        vtgate_socket_file = environment.tmproot + '/vtgate.sock'
        vtgate_server, vtgate_port = utils.vtgate_start(
            socket_file=vtgate_socket_file)

        master_tablet.start_vttablet()
        replica_tablet.start_vttablet()
        utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
        utils.check_db_read_write(master_tablet.tablet_uid)

        for t in [master_tablet, replica_tablet]:
            t.reset_replication()
        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/0',
            master_tablet.tablet_alias
        ],
                        auto_log=True)

        # reset counter so tests don't assert
        tablet.Tablet.tablets_running = 0

        master_start_position = _get_master_current_position()
        master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
        master_tablet.mquery('vt_test_keyspace', _create_vt_a)
        master_tablet.mquery('vt_test_keyspace', _create_vt_b)

        utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
        utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

    except:
        tearDownModule()
        raise
Beispiel #25
0
    def _test_reparent_slave_offline(self, shard_id='0'):
        utils.run_vtctl('CreateKeyspace test_keyspace')

        # create the database so vttablets start, as they are serving
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62044.create_db('vt_test_keyspace')
        tablet_41983.create_db('vt_test_keyspace')
        tablet_31981.create_db('vt_test_keyspace')

        # Start up a master mysql and vttablet
        tablet_62344.init_tablet('master',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)

        # Create a few slaves for testing reparenting.
        tablet_62044.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_41983.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_31981.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)

        # wait for all tablets to start
        for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
            t.wait_for_vttablet_state("SERVING")

        # Recompute the shard layout node - until you do that, it might not be valid.
        utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id)
        utils.validate_topology()

        # Force the slaves to reparent assuming that all the datasets are identical.
        for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
            t.reset_replication()
        utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' %
                        (shard_id, tablet_62344.tablet_alias))
        utils.validate_topology(ping_tablets=True)

        self._check_db_addr(shard_id, 'master', tablet_62344.port)

        # Kill one tablet so we seem offline
        tablet_31981.kill_vttablet()

        # Perform a graceful reparent operation.
        utils.run_vtctl('ReparentShard test_keyspace/%s %s' %
                        (shard_id, tablet_62044.tablet_alias))

        tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983])
Beispiel #26
0
    def _test_sanity(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace'])
        utils.run_vtctl(['createshard', '-force', 'test_keyspace/0'])
        tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
        utils.run_vtctl(
            ['RebuildKeyspaceGraph', '-rebuild_srv_shards', 'test_keyspace'])
        utils.validate_topology()
        self._check_srv_shard()

        # if these statements don't run before the tablet it will wedge
        # waiting for the db to become accessible. this is more a bug than
        # a feature.
        tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                              self._populate_vt_select_test)

        tablet_62344.start_vttablet()

        # make sure the query service is started right away
        qr = tablet_62344.execute('select * from vt_select_test')
        self.assertEqual(len(qr['Rows']), 4,
                         'expected 4 rows in vt_select_test: %s' % str(qr))

        # make sure direct dba queries work
        query_result = utils.run_vtctl_json([
            'ExecuteFetchAsDba', tablet_62344.tablet_alias,
            'select * from vt_test_keyspace.vt_select_test'
        ])
        self.assertEqual(
            len(query_result['rows']), 4,
            'expected 4 rows in vt_select_test: %s' % str(query_result))
        self.assertEqual(
            len(query_result['fields']), 2,
            'expected 2 fields in vt_select_test: %s' % str(query_result))

        # check Ping / RefreshState
        utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
        utils.run_vtctl(['RefreshState', tablet_62344.tablet_alias])

        # Quickly check basic actions.
        utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias])
        utils.wait_db_read_only(62344)

        utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias])
        utils.check_db_read_write(62344)

        utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias])
        utils.wait_db_read_only(62344)

        utils.validate_topology()
        utils.run_vtctl(['ValidateKeyspace', 'test_keyspace'])
        # not pinging tablets, as it enables replication checks, and they
        # break because we only have a single master, no slaves
        utils.run_vtctl(
            ['ValidateShard', '-ping-tablets=false', 'test_keyspace/0'])
        self._check_srv_shard()

        tablet_62344.kill_vttablet()
def test_multisnapshot_and_restore_vtctl():
    tables = ["vt_insert_test", "vt_insert_test1"]
    create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB"""
    insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # Start three tablets for three different shards. At this point the
    # sharding schema is not really important, as long as it is
    # consistent.
    new_spec = "-0000000000000028-"
    old_tablets = [tablet_62044, tablet_41983, tablet_31981]
    for i, tablet in enumerate(old_tablets):
        tablet.init_tablet("master", "test_keyspace", str(i))
        utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
    utils.validate_topology()

    for i, tablet in enumerate(old_tablets):
        tablet.populate(
            "vt_test_keyspace",
            [create_template % table for table in tables],
            sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
        )
        tablet.start_vttablet()
        utils.run_vtctl(
            "MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id" % (new_spec, tablet.tablet_alias),
            trap_output=True,
        )

    utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
    tablet_62344.init_tablet("master", "test_keyspace_new", "-0000000000000028", dbname="not_vt_test_keyspace")
    utils.run_vtctl("RebuildShardGraph test_keyspace_new/-0000000000000028")
    utils.validate_topology()
    tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
    tablet_62344.start_vttablet(wait_for_state="CONNECTING")  # db not created

    # 0x28 = 40
    source_aliases = " ".join(t.tablet_alias for t in old_tablets)
    utils.run_vtctl(
        "MultiRestore %s %s" % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True
    )
    time.sleep(1)
    for table in tables:
        rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
        if len(rows) == 0:
            raise utils.TestError("There are no rows in the restored database.")
        for row in rows:
            if row[0] > 32:
                raise utils.TestError("Bad row: %s" % row)
    for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
        tablet.kill_vttablet()
Beispiel #28
0
def run_test_mysqlctl_split():
  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  tablet_62344.init_tablet('master', 'test_keyspace', '0')
  utils.run_vtctl('RebuildShardGraph test_keyspace/0')
  utils.validate_topology()

  tablet_62344.populate('vt_test_keyspace', create_vt_insert_test,
                        populate_vt_insert_test)

  tablet_62344.start_vttablet()

  err = tablet_62344.mysqlctl('-port %u -mysql-port %u partialsnapshot -end=0000000000000003 vt_test_keyspace id' % (tablet_62344.port, tablet_62344.mysql_port)).wait()
  if err != 0:
    raise utils.TestError('mysqlctl partialsnapshot failed')


  utils.pause("partialsnapshot finished")

  tablet_62044.mquery('', 'stop slave')
  tablet_62044.create_db('vt_test_keyspace')
  call(["touch", "/tmp/vtSimulateFetchFailures"])
  err = tablet_62044.mysqlctl('-port %u -mysql-port %u partialrestore %s/snapshot/vt_0000062344/data/vt_test_keyspace-,0000000000000003/partial_snapshot_manifest.json' % (tablet_62044.port, tablet_62044.mysql_port, utils.vtdataroot)).wait()
  if err != 0:
    raise utils.TestError('mysqlctl partialrestore failed')

  tablet_62044.assert_table_count('vt_test_keyspace', 'vt_insert_test', 2)

  # change/add two values on the master, one in range, one out of range, make
  # sure the right one propagate and not the other
  utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias)
  tablet_62344.mquery('vt_test_keyspace', "insert into vt_insert_test (id, msg) values (5, 'test should not propagate')", write=True)
  tablet_62344.mquery('vt_test_keyspace', "update vt_insert_test set msg='test should propagate' where id=2", write=True)

  utils.pause("look at db now!")

  # wait until value that should have been changed is here
  timeout = 10
  while timeout > 0:
    result = tablet_62044.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=2')
    if result[0][0] == "test should propagate":
      break
    timeout -= 1
    time.sleep(1)
  if timeout == 0:
    raise utils.TestError("expected propagation to happen", result)

  # test value that should not propagate
  # this part is disabled now, as the replication pruning is only enabled
  # for row-based replication, but the mysql server is statement based.
  # will re-enable once we get statement-based pruning patch into mysql.
#  tablet_62044.assert_table_count('vt_test_keyspace', 'vt_insert_test', 0, 'where id=5')

  tablet_62344.kill_vttablet()
Beispiel #29
0
    def test_actions_and_timeouts(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(["CreateKeyspace", "test_keyspace"])

        tablet_62344.init_tablet("master", "test_keyspace", "0")
        utils.run_vtctl(["RebuildShardGraph", "test_keyspace/0"])
        utils.validate_topology()
        self._check_srv_shard()
        tablet_62344.create_db("vt_test_keyspace")
        tablet_62344.start_vttablet()

        utils.run_vtctl(["Ping", tablet_62344.tablet_alias])

        # schedule long action in the background, sleep a little bit to make sure
        # it started to run
        args = (
            environment.binary_args("vtctl")
            + environment.topo_server().flags()
            + [
                "-tablet_manager_protocol",
                protocols_flavor().tablet_manager_protocol(),
                "-tablet_protocol",
                protocols_flavor().tabletconn_protocol(),
                "-log_dir",
                environment.vtlogroot,
                "Sleep",
                tablet_62344.tablet_alias,
                "10s",
            ]
        )
        bg = utils.run_bg(args)
        time.sleep(3)

        # try a frontend RefreshState that should timeout as the tablet is busy
        # running the other one
        _, stderr = utils.run_vtctl(["-wait-time", "3s", "RefreshState", tablet_62344.tablet_alias], expect_fail=True)
        self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

        # wait for the background vtctl
        bg.wait()

        if environment.topo_server().flavor() == "zookeeper":
            # extra small test: we ran for a while, get the states we were in,
            # make sure they're accounted for properly
            # first the query engine States
            v = utils.get_vars(tablet_62344.port)
            logging.debug("vars: %s", v)

            # then the Zookeeper connections
            if v["ZkCachedConn"]["test_nj"] != "Connected":
                self.fail("invalid zk test_nj state: %s" % v["ZkCachedConn"]["test_nj"])
            if v["ZkCachedConn"]["global"] != "Connected":
                self.fail("invalid zk global state: %s" % v["ZkCachedConn"]["global"])
            if v["TabletType"] != "master":
                self.fail("TabletType not exported correctly")

        tablet_62344.kill_vttablet()
  def _test_sanity(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace'])
    utils.run_vtctl(['createshard', '-force', 'test_keyspace/0'])
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl(
        ['RebuildKeyspaceGraph', '-rebuild_srv_shards', 'test_keyspace'])
    utils.validate_topology()
    self._check_srv_shard()

    # if these statements don't run before the tablet it will wedge
    # waiting for the db to become accessible. this is more a bug than
    # a feature.
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()

    # make sure the query service is started right away
    qr = tablet_62344.execute('select * from vt_select_test')
    self.assertEqual(len(qr['rows']), 4,
                     'expected 4 rows in vt_select_test: %s' % str(qr))

    # make sure direct dba queries work
    query_result = utils.run_vtctl_json(
        ['ExecuteFetchAsDba', '-json', tablet_62344.tablet_alias,
         'select * from vt_test_keyspace.vt_select_test'])
    self.assertEqual(
        len(query_result['rows']), 4,
        'expected 4 rows in vt_select_test: %s' % str(query_result))
    self.assertEqual(
        len(query_result['fields']), 2,
        'expected 2 fields in vt_select_test: %s' % str(query_result))

    # check Ping / RefreshState
    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])
    utils.run_vtctl(['RefreshState', tablet_62344.tablet_alias])

    # Quickly check basic actions.
    utils.run_vtctl(['SetReadOnly', tablet_62344.tablet_alias])
    utils.wait_db_read_only(62344)

    utils.run_vtctl(['SetReadWrite', tablet_62344.tablet_alias])
    utils.check_db_read_write(62344)

    utils.run_vtctl(['DemoteMaster', tablet_62344.tablet_alias])
    utils.wait_db_read_only(62344)

    utils.validate_topology()
    utils.run_vtctl(['ValidateKeyspace', 'test_keyspace'])
    # not pinging tablets, as it enables replication checks, and they
    # break because we only have a single master, no slaves
    utils.run_vtctl(['ValidateShard', '-ping-tablets=false',
                     'test_keyspace/0'])
    self._check_srv_shard()

    tablet_62344.kill_vttablet()
Beispiel #31
0
  def setUpClass(klass):
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl(['CreateKeyspace',
                     '--served-from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
                     'redirected_keyspace'])

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('spare',   'test_keyspace', '-80')
    shard_0_spare.init_tablet(  'spare',   'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
    idle.init_tablet('idle')
    scrap.init_tablet('idle')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
                    auto_log=True)

    # start running all the tablets
    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None,
                       extra_args=utils.vtctld.process_args())
    shard_0_replica.create_db('vt_test_keyspace')
    shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(),
                                   target_tablet_type='replica',
                                   wait_for_state=None)

    for t in scrap, idle, shard_0_spare:
      t.start_vttablet(wait_for_state=None,
                       extra_args=utils.vtctld.process_args())

    # wait for the right states
    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.wait_for_vttablet_state('SERVING')
    for t in [scrap, idle, shard_0_replica, shard_0_spare]:
      t.wait_for_vttablet_state('NOT_SERVING')

    scrap.scrap()

    for t in [shard_0_master, shard_0_replica, shard_0_spare,
              shard_1_master, shard_1_replica, idle, scrap]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-',
                     shard_1_master.tablet_alias], auto_log=True)
    shard_0_replica.wait_for_vttablet_state('SERVING')

    # run checks now before we start the tablets
    utils.validate_topology()

    # start a vtgate server too
    global vtgate_server, vtgate_port
    vtgate_server, vtgate_port = utils.vtgate_start(
        cache_ttl='0s', extra_args=utils.vtctld.process_args())
Beispiel #32
0
  def test_reparent_lag_slave(self, shard_id='0'):
    utils.run_vtctl('CreateKeyspace test_keyspace')

    # create the database so vttablets start, as they are serving
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62044.create_db('vt_test_keyspace')
    tablet_41983.create_db('vt_test_keyspace')
    tablet_31981.create_db('vt_test_keyspace')

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet('master', 'test_keyspace', shard_id, start=True, wait_for_start=False)

    # Create a few slaves for testing reparenting.
    tablet_62044.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False)
    tablet_31981.init_tablet('replica', 'test_keyspace', shard_id, start=True, wait_for_start=False)
    tablet_41983.init_tablet('lag', 'test_keyspace', shard_id, start=True, wait_for_start=False)

    # wait for all tablets to start
    for t in [tablet_62344, tablet_62044, tablet_31981]:
      t.wait_for_vttablet_state("SERVING")
    tablet_41983.wait_for_vttablet_state("NOT_SERVING")

    # Recompute the shard layout node - until you do that, it might not be valid.
    utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id)
    utils.validate_topology()

    # Force the slaves to reparent assuming that all the datasets are identical.
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
      t.reset_replication()
    utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' % (shard_id, tablet_62344.tablet_alias))
    utils.validate_topology(ping_tablets=True)

    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.mquery('vt_test_keyspace', self._create_vt_insert_test)

    tablet_41983.mquery('', 'stop slave')
    for q in self._populate_vt_insert_test:
      tablet_62344.mquery('vt_test_keyspace', q, write=True)

    # Perform a graceful reparent operation.
    utils.run_vtctl('ReparentShard test_keyspace/%s %s' % (shard_id, tablet_62044.tablet_alias))

    tablet_41983.mquery('', 'start slave')
    time.sleep(1)

    utils.pause("check orphan")

    utils.run_vtctl('ReparentTablet %s' % tablet_41983.tablet_alias)

    result = tablet_41983.mquery('vt_test_keyspace', 'select msg from vt_insert_test where id=1')
    if len(result) != 1:
      self.fail('expected 1 row from vt_insert_test: %s' % str(result))

    utils.pause("check lag reparent")

    tablet.kill_tablets([tablet_62344, tablet_62044, tablet_41983, tablet_31981])
Beispiel #33
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    self._check_srv_shard()
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server().flags() +
            ['-tablet_manager_protocol',
             protocols_flavor().tablet_manager_protocol(),
             '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
             '-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RefreshState that should timeout as the tablet is busy
    # running the other one
    stdout, stderr = utils.run_vtctl(
        ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
        expect_fail=True)
    self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

    # wait for the background vtctl
    bg.wait()

    if environment.topo_server().flavor() == 'zookeeper':
      # extra small test: we ran for a while, get the states we were in,
      # make sure they're accounted for properly
      # first the query engine States
      v = utils.get_vars(tablet_62344.port)
      logging.debug('vars: %s', v)

      # then the Zookeeper connections
      if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
        self.fail('invalid zk test_nj state: %s' %
                  v['ZkMetaConn']['test_nj']['Current'])
      if v['ZkMetaConn']['global']['Current'] != 'Connected':
        self.fail('invalid zk global state: %s' %
                  v['ZkMetaConn']['global']['Current'])
      if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
        self.fail('not enough time in Connected state: %d',
                  v['ZkMetaConn']['test_nj']['DurationConnected'])
      if v['TabletType'] != 'master':
        self.fail('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
Beispiel #34
0
def setUpModule():
  global vtgate_server
  global vtgate_port
  global vtgate_socket_file
  global master_start_position

  try:
    environment.topo_server_setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()
                   ]
    utils.wait_procs(setup_procs)

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    master_tablet.create_db('vt_test_keyspace')
    master_tablet.create_db('other_database')
    replica_tablet.create_db('vt_test_keyspace')
    replica_tablet.create_db('other_database')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

    vtgate_socket_file = environment.tmproot + '/vtgate.sock'
    vtgate_server, vtgate_port = utils.vtgate_start(socket_file=vtgate_socket_file)

    master_tablet.start_vttablet()
    replica_tablet.start_vttablet()
    utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
    utils.check_db_read_write(master_tablet.tablet_uid)

    for t in [master_tablet, replica_tablet]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)

    # reset counter so tests don't assert
    tablet.Tablet.tablets_running = 0

    master_start_position = _get_master_current_position()
    master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
    master_tablet.mquery('vt_test_keyspace', _create_vt_a)
    master_tablet.mquery('vt_test_keyspace', _create_vt_b)

    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

  except:
    tearDownModule()
    raise
Beispiel #35
0
  def test_vtgate(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace -force test_keyspace')
    utils.run_vtctl('CreateShard -force test_keyspace/0')
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
    utils.validate_topology()

    # if these statements don't run before the tablet it will wedge waiting for the
    # db to become accessible. this is more a bug than a feature.
    tablet_62344.mquery("", ["set global read_only = off"])
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()
    gate_proc = utils.vtgate_start()

    try:
      conn = vtgate.connect("localhost:%s"%(utils.vtgate_port_base), "master", "test_keyspace", "0", 2.0)

      # _execute
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))

      # _stream_execute
      (result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {})
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
      count = 0
      while 1:
        r = conn._stream_next()
        if not r:
          break
        count += 1
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-rollback
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.rollback()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-commit
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.commit()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 5, "want 5, got %d" % (count))

      # close
      conn.close()
    finally:
      utils.vtgate_kill(gate_proc)
      tablet_62344.kill_vttablet()
def run_test_reparent_lag_slave(shard_id="0"):
    utils.zk_wipe()

    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # create the database so vttablets start, as they are serving
    tablet_62344.create_db("vt_test_keyspace")
    tablet_62044.create_db("vt_test_keyspace")
    tablet_41983.create_db("vt_test_keyspace")
    tablet_31981.create_db("vt_test_keyspace")

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet("master", "test_keyspace", shard_id, start=True)

    # Create a few slaves for testing reparenting.
    tablet_62044.init_tablet("replica", "test_keyspace", shard_id, start=True)
    tablet_31981.init_tablet("replica", "test_keyspace", shard_id, start=True)
    tablet_41983.init_tablet("lag", "test_keyspace", shard_id, start=True)

    # Recompute the shard layout node - until you do that, it might not be valid.
    utils.run_vtctl("RebuildShardGraph test_keyspace/" + shard_id)
    utils.validate_topology()

    # Force the slaves to reparent assuming that all the datasets are identical.
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
        t.reset_replication()
    utils.run_vtctl("ReparentShard -force test_keyspace/%s %s" % (shard_id, tablet_62344.tablet_alias))
    utils.validate_topology(ping_tablets=True)

    tablet_62344.create_db("vt_test_keyspace")
    tablet_62344.mquery("vt_test_keyspace", create_vt_insert_test)

    tablet_41983.mquery("", "stop slave")
    for q in populate_vt_insert_test:
        tablet_62344.mquery("vt_test_keyspace", q, write=True)

    # Perform a graceful reparent operation.
    utils.run_vtctl("ReparentShard test_keyspace/%s %s" % (shard_id, tablet_62044.tablet_alias))

    tablet_41983.mquery("", "start slave")
    time.sleep(1)

    utils.pause("check orphan")

    utils.run_vtctl("ReparentTablet %s" % tablet_41983.tablet_alias)

    result = tablet_41983.mquery("vt_test_keyspace", "select msg from vt_insert_test where id=1")
    if len(result) != 1:
        raise utils.TestError("expected 1 row from vt_insert_test", result)

    utils.pause("check lag reparent")

    tablet_62344.kill_vttablet()
    tablet_62044.kill_vttablet()
    tablet_41983.kill_vttablet()
    tablet_31981.kill_vttablet()
Beispiel #37
0
def setUpModule():
  try:
    environment.topo_server().setup()

    _init_mysql(tablets)

    utils.run_vtctl(['CreateKeyspace', test_keyspace])

    shard_0_master.init_tablet(  'master',  test_keyspace, '0')
    shard_0_replica1.init_tablet('replica', test_keyspace, '0')
    shard_0_replica2.init_tablet('replica', test_keyspace, '0')
    shard_0_rdonly.init_tablet(  'rdonly',  test_keyspace, '0')
    shard_0_backup.init_tablet(  'backup',  test_keyspace, '0')
    shard_1_master.init_tablet(  'master',  test_keyspace, '1')
    shard_1_replica1.init_tablet('replica', test_keyspace, '1')

    utils.run_vtctl(['RebuildKeyspaceGraph', test_keyspace], auto_log=True)

    # run checks now before we start the tablets
    utils.validate_topology()

    utils.Vtctld().start()

    # create databases, start the tablets
    for t in tablets:
      t.create_db(db_name)
      t.start_vttablet(wait_for_state=None)

    # wait for the tablets to start
    shard_0_master.wait_for_vttablet_state('SERVING')
    shard_0_replica1.wait_for_vttablet_state('SERVING')
    shard_0_replica2.wait_for_vttablet_state('SERVING')
    shard_0_rdonly.wait_for_vttablet_state('SERVING')
    shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
    shard_1_master.wait_for_vttablet_state('SERVING')
    shard_1_replica1.wait_for_vttablet_state('SERVING')

    # make sure all replication is good
    for t in tablets:
      t.reset_replication()

    utils.run_vtctl(['InitShardMaster', test_keyspace+'/0',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['InitShardMaster', test_keyspace+'/1',
                     shard_1_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace])

    # check after all tablets are here and replication is fixed
    utils.validate_topology(ping_tablets=True)
  except Exception as setup_exception:
    try:
      tearDownModule()
    except Exception as e:
      logging.exception("Tearing down a failed setUpModule() failed: %s", e)
    raise setup_exception
Beispiel #38
0
    def _test_vtctl_clone(self, server_mode):
        if server_mode:
            clone_flags = '-server-mode'
        else:
            clone_flags = ''

        # Start up a master mysql and vttablet
        utils.run_vtctl('CreateKeyspace snapshot_test')

        tablet_62344.init_tablet('master', 'snapshot_test', '0')
        utils.run_vtctl('RebuildShardGraph snapshot_test/0')
        utils.validate_topology()

        tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test,
                              self._populate_vt_insert_test)
        tablet_62344.start_vttablet()

        tablet_62044.create_db('vt_snapshot_test')
        tablet_62044.init_tablet('idle', start=True)

        # small test to make sure the directory validation works
        snapshot_dir = os.path.join(utils.vtdataroot, 'snapshot')
        utils.run("rm -rf %s" % snapshot_dir)
        utils.run("mkdir -p %s" % snapshot_dir)
        utils.run("chmod -w %s" % snapshot_dir)
        out, err = utils.run(
            utils.vtroot +
            '/bin/vtctl --alsologtostderr Clone -force %s %s %s' %
            (clone_flags, tablet_62344.tablet_alias,
             tablet_62044.tablet_alias),
            trap_output=True,
            raise_on_error=False)
        if "Cannot validate snapshot directory" not in err:
            raise utils.TestError("expected validation error", err)
        if "Un-reserved test_nj-0000062044" not in err:
            raise utils.TestError("expected Un-reserved", err)
        logging.debug("Failed Clone output: " + err)
        utils.run("chmod +w %s" % snapshot_dir)

        call(["touch", "/tmp/vtSimulateFetchFailures"])
        utils.run_vtctl('Clone -force %s %s %s' %
                        (clone_flags, tablet_62344.tablet_alias,
                         tablet_62044.tablet_alias),
                        auto_log=True)

        utils.pause("look at logs!")
        tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test',
                                        4)
        tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test',
                                        4)

        utils.validate_topology()

        tablet_62344.kill_vttablet()
        tablet_62044.kill_vttablet()
Beispiel #39
0
  def test_topocustomrule(self):
    # Empty rule file.
    topocustomrule_file = environment.tmproot+'/rules.json'
    with open(topocustomrule_file, 'w') as fd:
      fd.write('[]\n')

    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace'])
    utils.run_vtctl(['createshard', '-force', 'test_keyspace/0'])
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
    utils.validate_topology()

    # Copy config file into topo.
    topocustomrule_path = '/keyspaces/test_keyspace/configs/CustomRules'
    utils.run_vtctl(['TopoCp', '-to_topo', topocustomrule_file,
                     topocustomrule_path])

    # Put some data in, start master.
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)
    tablet_62344.start_vttablet(topocustomrule_path=topocustomrule_path)

    # make sure the query service is working
    qr = tablet_62344.execute('select id, msg from vt_select_test')
    self.assertEqual(len(qr['rows']), 4,
                     'expected 4 rows in vt_select_test: %s' % str(qr))

    # Now update the topocustomrule file.
    with open(topocustomrule_file, 'w') as fd:
      fd.write('''
        [{
          "Name": "rule1",
          "Description": "disallow select on table vt_select_test",
          "TableNames" : ["vt_select_test"],
          "Query" : "(select)|(SELECT)"
        }]''')
    utils.run_vtctl(['TopoCp', '-to_topo', topocustomrule_file,
                     topocustomrule_path])

    # And wait until the query fails with the right error.
    timeout = 10.0
    while True:
      try:
        tablet_62344.execute('select id, msg from vt_select_test')
        timeout = utils.wait_step('query rule in place', timeout)
      except Exception as e:
        print e
        expected = ('disallowed due to rule: disallow select'
                    ' on table vt_select_test')
        self.assertIn(expected, str(e))
        break

    # Cleanup.
    tablet_62344.kill_vttablet()
Beispiel #40
0
  def test_topocustomrule(self):
    # Empty rule file.
    topocustomrule_file = environment.tmproot+'/rules.json'
    with open(topocustomrule_file, 'w') as fd:
      fd.write('[]\n')

    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', '-force', 'test_keyspace'])
    utils.run_vtctl(['createshard', '-force', 'test_keyspace/0'])
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
    utils.validate_topology()

    # Copy config file into topo.
    topocustomrule_path = '/keyspaces/test_keyspace/configs/CustomRules'
    utils.run_vtctl(['TopoCp', '-to_topo', topocustomrule_file,
                     topocustomrule_path])

    # Put some data in, start master.
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)
    tablet_62344.start_vttablet(topocustomrule_path=topocustomrule_path)

    # make sure the query service is working
    qr = tablet_62344.execute('select id, msg from vt_select_test')
    self.assertEqual(len(qr['rows']), 4,
                     'expected 4 rows in vt_select_test: %s' % str(qr))

    # Now update the topocustomrule file.
    with open(topocustomrule_file, 'w') as fd:
      fd.write('''
        [{
          "Name": "rule1",
          "Description": "disallow select on table vt_select_test",
          "TableNames" : ["vt_select_test"],
          "Query" : "(select)|(SELECT)"
        }]''')
    utils.run_vtctl(['TopoCp', '-to_topo', topocustomrule_file,
                     topocustomrule_path])

    # And wait until the query fails with the right error.
    timeout = 10.0
    while True:
      try:
        tablet_62344.execute('select id, msg from vt_select_test')
        timeout = utils.wait_step('query rule in place', timeout)
      except Exception as e:
        print e
        expected = ('disallowed due to rule: disallow select'
                    ' on table vt_select_test')
        self.assertIn(expected, str(e))
        break

    # Cleanup.
    tablet_62344.kill_vttablet()
    def test_reparent(self):
        name = 'sharded_message'
        keyspace = 'user'

        # Start a stream. Use a timeout that's greater than how long
        # the test will take to run.
        vtgate_conn = get_connection(120)
        (it, fields) = vtgate_conn.message_stream(keyspace, name)
        self.assertEqual(get_client_count(shard_0_master), 1)
        self.assertEqual(get_client_count(shard_0_replica), 0)
        self.assertEqual(get_client_count(shard_1_master), 1)

        # Perform a graceful reparent to the replica.
        utils.run_vtctl([
            'PlannedReparentShard', '-keyspace_shard', 'user/-80',
            '-new_master', shard_0_replica.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology()

        # Verify connection has migrated.
        # The wait must be at least 6s which is how long vtgate will
        # wait before retrying: that is 30s/5 where 30s is the default
        # message_stream_grace_period.
        time.sleep(10)
        self.assertEqual(get_client_count(shard_0_master), 0)
        self.assertEqual(get_client_count(shard_0_replica), 1)
        self.assertEqual(get_client_count(shard_1_master), 1)

        cursor = vtgate_conn.cursor(tablet_type='master',
                                    keyspace=None,
                                    writable=True)
        query = 'insert into %s(id, message) values(:id, :message)' % (name)
        cursor.begin()
        cursor.execute(query, {'id': 3, 'message': 'hello world 3'})
        cursor.commit()

        # Receive the message.
        it.next()

        # Reparent back to old master.
        utils.run_vtctl([
            'PlannedReparentShard', '-keyspace_shard', 'user/-80',
            '-new_master', shard_0_master.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology()

        time.sleep(10)
        self.assertEqual(get_client_count(shard_0_master), 1)
        self.assertEqual(get_client_count(shard_0_replica), 0)
        self.assertEqual(get_client_count(shard_1_master), 1)

        # Ack the message.
        count = vtgate_conn.message_ack(name, [3])
def run_test_scrap():
    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    tablet_62344.init_tablet("master", "test_keyspace", "0")
    tablet_62044.init_tablet("replica", "test_keyspace", "0")
    utils.run_vtctl("RebuildShardGraph test_keyspace/0")
    utils.validate_topology()

    tablet_62044.scrap(force=True)
    utils.validate_topology()
Beispiel #43
0
def setUpModule():
  global master_start_position

  try:
    environment.topo_server().setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()]
    utils.wait_procs(setup_procs)

    # start a vtctld so the vtctl insert commands are just RPCs, not forks
    utils.Vtctld().start()

    # Start up a master mysql and vttablet
    logging.debug('Setting up tablets')
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    master_tablet.create_db('vt_test_keyspace')
    master_tablet.create_db('other_database')
    replica_tablet.create_db('vt_test_keyspace')
    replica_tablet.create_db('other_database')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

    utils.VtGate().start()

    master_tablet.start_vttablet()
    replica_tablet.start_vttablet()
    utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
    utils.check_db_read_write(master_tablet.tablet_uid)

    for t in [master_tablet, replica_tablet]:
      t.reset_replication()
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)

    # reset counter so tests don't assert
    tablet.Tablet.tablets_running = 0

    master_start_position = _get_master_current_position()
    master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
    master_tablet.mquery('vt_test_keyspace', _create_vt_a)
    master_tablet.mquery('vt_test_keyspace', _create_vt_b)

    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

  except:
    tearDownModule()
    raise
Beispiel #44
0
def run_test_scrap():
  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  tablet_62344.init_tablet('master', 'test_keyspace', '0')
  tablet_62044.init_tablet('replica', 'test_keyspace', '0')
  utils.run_vtctl('RebuildShardGraph test_keyspace/0')
  utils.validate_topology()

  tablet_62044.scrap(force=True)
  utils.validate_topology()
Beispiel #45
0
  def test_scrap(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    tablet_62044.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/*'])
    utils.validate_topology()

    tablet_62044.scrap(force=True)
    utils.validate_topology()
Beispiel #46
0
    def _test_mysqlctl_clone(server_mode):
        if server_mode:
            snapshot_cmd = ['snapshotsourcestart', '-concurrency=8']
            restore_flags = ['-dont_wait_for_slave_start']
        else:
            snapshot_cmd = ['snapshot', '-concurrency=5']
            restore_flags = []

        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'snapshot_test'])

        tablet_62344.init_tablet('master', 'snapshot_test', '0')
        utils.run_vtctl(['RebuildShardGraph', 'snapshot_test/0'])
        utils.validate_topology()

        tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test,
                              self._populate_vt_insert_test)

        tablet_62344.start_vttablet()

        err = tablet_62344.mysqlctl(snapshot_cmd + ['vt_snapshot_test'],
                                    with_ports=True).wait()
        if err != 0:
            self.fail('mysqlctl %s failed' % str(snapshot_cmd))

        utils.pause("%s finished" % str(snapshot_cmd))

        call(["touch", "/tmp/vtSimulateFetchFailures"])
        err = tablet_31981.mysqlctl(
            ['restore', '-fetch_concurrency=2', '-fetch_retry_count=4'] +
            restore_flags + [
                environment.vtdataroot +
                '/snapshot/vt_0000062344/snapshot_manifest.json'
            ],
            with_ports=True).wait()
        if err != 0:
            self.fail('mysqlctl restore failed')

        tablet_31981.assert_table_count('vt_snapshot_test', 'vt_insert_test',
                                        4)

        if server_mode:
            err = tablet_62344.mysqlctl(
                ['snapshotsourceend', '-read_write', 'vt_snapshot_test'],
                with_ports=True).wait()
            if err != 0:
                self.fail('mysqlctl snapshotsourceend failed')

            # see if server restarted properly
            tablet_62344.assert_table_count('vt_snapshot_test',
                                            'vt_insert_test', 4)

        tablet_62344.kill_vttablet()
Beispiel #47
0
def _run_test_mysqlctl_clone(server_mode):
    if server_mode:
        snapshot_cmd = "snapshotsourcestart -concurrency=8"
        restore_flags = "-dont-wait-for-slave-start"
    else:
        snapshot_cmd = "snapshot -concurrency=5"
        restore_flags = ""

    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace -force snapshot_test')

    tablet_62344.init_tablet('master', 'snapshot_test', '0')
    utils.run_vtctl('RebuildShardGraph snapshot_test/0')
    utils.validate_topology()

    tablet_62344.populate('vt_snapshot_test', create_vt_insert_test,
                          populate_vt_insert_test)

    tablet_62344.start_vttablet()

    err = tablet_62344.mysqlctl(
        '-port %u -mysql-port %u %s vt_snapshot_test' %
        (tablet_62344.port, tablet_62344.mysql_port, snapshot_cmd)).wait()
    if err != 0:
        raise utils.TestError('mysqlctl %s failed' % snapshot_cmd)

    utils.pause("%s finished" % snapshot_cmd)

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    err = tablet_62044.mysqlctl(
        '-port %u -mysql-port %u restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s/snapshot/vt_0000062344/snapshot_manifest.json'
        % (tablet_62044.port, tablet_62044.mysql_port, restore_flags,
           utils.vtdataroot)).wait()
    if err != 0:
        raise utils.TestError('mysqlctl restore failed')

    tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)

    if server_mode:
        err = tablet_62344.mysqlctl(
            '-port %u -mysql-port %u snapshotsourceend -read-write vt_snapshot_test'
            % (tablet_62344.port, tablet_62344.mysql_port)).wait()
        if err != 0:
            raise utils.TestError('mysqlctl snapshotsourceend failed')

        # see if server restarted properly
        tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test',
                                        4)

    tablet_62344.kill_vttablet()
Beispiel #48
0
def setUpModule():
    try:
        environment.topo_server().setup()
        setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
        utils.wait_procs(setup_procs)

        # start a vtctld so the vtctl insert commands are just RPCs, not forks.
        utils.Vtctld().start()

        # Start up a master mysql and vttablet
        logging.debug('Setting up tablets')
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        master_tablet.init_tablet('master',
                                  'test_keyspace',
                                  '0',
                                  tablet_index=0)
        replica_tablet.init_tablet('replica',
                                   'test_keyspace',
                                   '0',
                                   tablet_index=1)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                        auto_log=True)
        utils.validate_topology()
        master_tablet.create_db('vt_test_keyspace')
        replica_tablet.create_db('vt_test_keyspace')

        master_tablet.start_vttablet(wait_for_state=None)
        replica_tablet.start_vttablet(wait_for_state=None)
        master_tablet.wait_for_vttablet_state('SERVING')
        replica_tablet.wait_for_vttablet_state('NOT_SERVING')
        utils.run_vtctl(
            ['InitShardMaster', 'test_keyspace/0', master_tablet.tablet_alias],
            auto_log=True)

        utils.wait_for_tablet_type(replica_tablet.tablet_alias, 'replica')
        master_tablet.wait_for_vttablet_state('SERVING')
        replica_tablet.wait_for_vttablet_state('SERVING')

        master_tablet.mquery('vt_test_keyspace', _create_vt_a)
        master_tablet.mquery('vt_test_keyspace', _create_vt_b)

        utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
        utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])
        utils.run_vtctl(['RebuildVSchemaGraph'])

        utils.VtGate().start(tablets=[master_tablet, replica_tablet])
        utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
        utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)

    except:
        tearDownModule()
        raise
Beispiel #49
0
  def test_restart_during_action(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace test_keyspace')

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.validate_topology()
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl('Ping ' + tablet_62344.tablet_alias)

    # schedule long action
    utils.run_vtctl('-no-wait Sleep %s 15s' % tablet_62344.tablet_alias, stdout=utils.devnull)
    # ping blocks until the sleep finishes unless we have a schedule race
    action_path, _ = utils.run_vtctl('-no-wait Ping ' + tablet_62344.tablet_alias, trap_output=True)

    # kill agent leaving vtaction running
    tablet_62344.kill_vttablet()

    # restart agent
    tablet_62344.start_vttablet()

    # we expect this action with a short wait time to fail. this isn't the best
    # and has some potential for flakiness.
    utils.run_vtctl('-wait-time 2s WaitForAction ' + action_path, expect_fail=True)

    # wait until the background sleep action is done, otherwise there will be
    # a leftover vtaction whose result may overwrite running actions
    # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug:
    # the zombie Sleep clobbers the Clone command in the following tests
    utils.run_vtctl('-wait-time 20s WaitForAction ' + action_path,
                    auto_log=True)

    # extra small test: we ran for a while, get the states we were in,
    # make sure they're accounted for properly
    # first the query engine States
    v = utils.get_vars(tablet_62344.port)
    logging.debug("vars: %s" % str(v))
    if v['Voltron']['States']['DurationSERVING'] < 10e9:
      raise utils.TestError('not enough time in Open state', v['Voltron']['States']['DurationSERVING'])
    # then the Zookeeper connections
    if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
      raise utils.TestError('invalid zk test_nj state: ', v['ZkMetaConn']['test_nj']['Current'])
    if v['ZkMetaConn']['global']['Current'] != 'Connected':
      raise utils.TestError('invalid zk global state: ', v['ZkMetaConn']['global']['Current'])
    if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
      raise utils.TestError('not enough time in Connected state', v['ZkMetaConn']['test_nj']['DurationConnected'])
    if v['TabletType'] != 'master':
      raise utils.TestError('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
  def test_restart_during_action(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace test_keyspace')

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.validate_topology()
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl('Ping ' + tablet_62344.tablet_alias)

    # schedule long action
    utils.run_vtctl('-no-wait Sleep %s 15s' % tablet_62344.tablet_alias, stdout=utils.devnull)
    # ping blocks until the sleep finishes unless we have a schedule race
    action_path, _ = utils.run_vtctl('-no-wait Ping ' + tablet_62344.tablet_alias, trap_output=True)

    # kill agent leaving vtaction running
    tablet_62344.kill_vttablet()

    # restart agent
    tablet_62344.start_vttablet()

    # we expect this action with a short wait time to fail. this isn't the best
    # and has some potential for flakiness.
    utils.run_fail(utils.vtroot+'/bin/vtctl -log_dir '+utils.tmp_root+' --alsologtostderr -wait-time 2s WaitForAction ' + action_path)

    # wait until the background sleep action is done, otherwise there will be
    # a leftover vtaction whose result may overwrite running actions
    # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug:
    # the zombie Sleep clobbers the Clone command in the following tests
    utils.run_vtctl('-wait-time 20s WaitForAction ' + action_path,
                    auto_log=True)

    # extra small test: we ran for a while, get the states we were in,
    # make sure they're accounted for properly
    # first the query engine States
    v = utils.get_vars(tablet_62344.port)
    logging.debug("vars: %s" % str(v))
    if v['Voltron']['States']['DurationSERVING'] < 10e9:
      raise utils.TestError('not enough time in Open state', v['Voltron']['States']['DurationSERVING'])
    # then the Zookeeper connections
    if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
      raise utils.TestError('invalid zk test_nj state: ', v['ZkMetaConn']['test_nj']['Current'])
    if v['ZkMetaConn']['global']['Current'] != 'Connected':
      raise utils.TestError('invalid zk global state: ', v['ZkMetaConn']['global']['Current'])
    if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
      raise utils.TestError('not enough time in Connected state', v['ZkMetaConn']['test_nj']['DurationConnected'])
    if v['TabletType'] != 'master':
      raise utils.TestError('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
Beispiel #51
0
    def _test_vtctl_clone(self, server_mode):
        if server_mode:
            clone_flags = ['-server-mode']
        else:
            clone_flags = []

        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'snapshot_test'])

        tablet_62344.init_tablet('master', 'snapshot_test', '0')
        utils.run_vtctl(['RebuildShardGraph', 'snapshot_test/0'])
        utils.validate_topology()

        tablet_62344.populate('vt_snapshot_test', self._create_vt_insert_test,
                              self._populate_vt_insert_test)
        tablet_62344.start_vttablet()

        tablet_31981.create_db('vt_snapshot_test')
        tablet_31981.init_tablet('idle', start=True)

        # small test to make sure the directory validation works
        snapshot_dir = os.path.join(environment.vtdataroot, 'snapshot')
        utils.run("rm -rf %s" % snapshot_dir)
        utils.run("mkdir -p %s" % snapshot_dir)
        utils.run("chmod -w %s" % snapshot_dir)
        out, err = utils.run_vtctl(
            ['Clone', '-force'] + clone_flags +
            [tablet_62344.tablet_alias, tablet_31981.tablet_alias],
            log_level='INFO',
            expect_fail=True)
        if "Cannot validate snapshot directory" not in err:
            self.fail("expected validation error: %s" % err)
        if "Un-reserved test_ny-0000031981" not in err:
            self.fail("expected Un-reserved: %s" % err)
        logging.debug("Failed Clone output: " + err)
        utils.run("chmod +w %s" % snapshot_dir)

        call(["touch", "/tmp/vtSimulateFetchFailures"])
        utils.run_vtctl(['Clone', '-force'] + clone_flags +
                        [tablet_62344.tablet_alias, tablet_31981.tablet_alias],
                        auto_log=True)
        self._check_shard()

        utils.pause("look at logs!")
        tablet_31981.assert_table_count('vt_snapshot_test', 'vt_insert_test',
                                        4)
        tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test',
                                        4)

        utils.validate_topology()

        tablet.kill_tablets([tablet_62344, tablet_31981])
def _run_test_vtctl_clone(server_mode):
    if server_mode:
        clone_flags = "-server-mode"
    else:
        clone_flags = ""
    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force snapshot_test")

    tablet_62344.init_tablet("master", "snapshot_test", "0")
    utils.run_vtctl("RebuildShardGraph snapshot_test/0")
    utils.validate_topology()

    tablet_62344.populate("vt_snapshot_test", create_vt_insert_test, populate_vt_insert_test)
    tablet_62344.start_vttablet()

    tablet_62044.create_db("vt_snapshot_test")
    tablet_62044.init_tablet("idle", start=True)

    # small test to make sure the directory validation works
    snapshot_dir = os.path.join(utils.vtdataroot, "snapshot")
    utils.run("rm -rf %s" % snapshot_dir)
    utils.run("mkdir -p %s" % snapshot_dir)
    utils.run("chmod -w %s" % snapshot_dir)
    out, err = utils.run(
        utils.vtroot
        + "/bin/vtctl --alsologtostderr Clone -force %s %s %s"
        % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias),
        trap_output=True,
        raise_on_error=False,
    )
    if "Cannot validate snapshot directory" not in err:
        raise utils.TestError("expected validation error", err)
    if "Un-reserved test_nj-0000062044" not in err:
        raise utils.TestError("expected Un-reserved", err)
    utils.debug("Failed Clone output: " + err)
    utils.run("chmod +w %s" % snapshot_dir)

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    utils.run_vtctl(
        "Clone -force %s %s %s" % (clone_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias), auto_log=True
    )

    utils.pause("look at logs!")
    tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)
    tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

    utils.validate_topology()

    tablet_62344.kill_vttablet()
    tablet_62044.kill_vttablet()
Beispiel #53
0
    def test_scrap(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        tablet_62344.init_tablet('master', 'test_keyspace', '0')
        tablet_62044.init_tablet('replica', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/*'])
        utils.validate_topology()
        self._check_srv_shard()

        tablet_62044.scrap(force=True)
        utils.validate_topology()
        self._check_srv_shard()
def _run_test_mysqlctl_clone(server_mode):
    if server_mode:
        snapshot_cmd = "snapshotsourcestart -concurrency=8"
        restore_flags = "-dont-wait-for-slave-start"
    else:
        snapshot_cmd = "snapshot -concurrency=5"
        restore_flags = ""

    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force snapshot_test")

    tablet_62344.init_tablet("master", "snapshot_test", "0")
    utils.run_vtctl("RebuildShardGraph snapshot_test/0")
    utils.validate_topology()

    tablet_62344.populate("vt_snapshot_test", create_vt_insert_test, populate_vt_insert_test)

    tablet_62344.start_vttablet()

    err = tablet_62344.mysqlctl(
        "-port %u -mysql-port %u %s vt_snapshot_test" % (tablet_62344.port, tablet_62344.mysql_port, snapshot_cmd)
    ).wait()
    if err != 0:
        raise utils.TestError("mysqlctl %s failed" % snapshot_cmd)

    utils.pause("%s finished" % snapshot_cmd)

    call(["touch", "/tmp/vtSimulateFetchFailures"])
    err = tablet_62044.mysqlctl(
        "-port %u -mysql-port %u restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s/snapshot/vt_0000062344/snapshot_manifest.json"
        % (tablet_62044.port, tablet_62044.mysql_port, restore_flags, utils.vtdataroot)
    ).wait()
    if err != 0:
        raise utils.TestError("mysqlctl restore failed")

    tablet_62044.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

    if server_mode:
        err = tablet_62344.mysqlctl(
            "-port %u -mysql-port %u snapshotsourceend -read-write vt_snapshot_test"
            % (tablet_62344.port, tablet_62344.mysql_port)
        ).wait()
        if err != 0:
            raise utils.TestError("mysqlctl snapshotsourceend failed")

        # see if server restarted properly
        tablet_62344.assert_table_count("vt_snapshot_test", "vt_insert_test", 4)

    tablet_62344.kill_vttablet()
def run_test_restart_during_action():
    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    tablet_62344.init_tablet("master", "test_keyspace", "0")
    utils.run_vtctl("RebuildShardGraph test_keyspace/0")
    utils.validate_topology()
    tablet_62344.create_db("vt_test_keyspace")
    tablet_62344.start_vttablet()

    utils.run_vtctl("Ping " + tablet_62344.tablet_alias)

    # schedule long action
    utils.run_vtctl("-no-wait Sleep %s 15s" % tablet_62344.tablet_alias, stdout=devnull)
    # ping blocks until the sleep finishes unless we have a schedule race
    action_path, _ = utils.run_vtctl("-no-wait Ping " + tablet_62344.tablet_alias, trap_output=True)

    # kill agent leaving vtaction running
    tablet_62344.kill_vttablet()

    # restart agent
    tablet_62344.start_vttablet()

    # we expect this action with a short wait time to fail. this isn't the best
    # and has some potential for flakiness.
    utils.run_fail(utils.vtroot + "/bin/vtctl --alsologtostderr -wait-time 2s WaitForAction " + action_path)

    # wait until the background sleep action is done, otherwise there will be
    # a leftover vtaction whose result may overwrite running actions
    # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug:
    # the zombie Sleep clobbers the Clone command in the following tests
    utils.run(utils.vtroot + "/bin/vtctl --alsologtostderr -wait-time 20s WaitForAction " + action_path)

    # extra small test: we ran for a while, get the states we were in,
    # make sure they're accounted for properly
    # first the query engine States
    v = utils.get_vars(tablet_62344.port)
    utils.debug("vars: %s" % str(v))
    if v["Voltron"]["States"]["DurationOPEN"] < 10e9:
        raise utils.TestError("not enough time in Open state", v["Voltron"]["States"]["DurationOPEN"])
    # then the Zookeeper connections
    if v["ZkMetaConn"]["test_nj"]["Current"] != "Connected":
        raise utils.TestError("invalid zk test_nj state: ", v["ZkMetaConn"]["test_nj"]["Current"])
    if v["ZkMetaConn"]["global"]["Current"] != "Connected":
        raise utils.TestError("invalid zk global state: ", v["ZkMetaConn"]["global"]["Current"])
    if v["ZkMetaConn"]["test_nj"]["DurationConnected"] < 10e9:
        raise utils.TestError("not enough time in Connected state", v["ZkMetaConn"]["test_nj"]["DurationConnected"])
    if v["tablet-type"] != "master":
        raise utils.TestError("tablet-type not exported correctly")

    tablet_62344.kill_vttablet()
Beispiel #56
0
def test_multisnapshot_and_restore_vtctl():
  tables = ['vt_insert_test', 'vt_insert_test1']
  create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB'''
  insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  # Start three tablets for three different shards. At this point the
  # sharding schema is not really important, as long as it is
  # consistent.
  new_spec = '-0000000000000028-'
  old_tablets = [tablet_62044, tablet_41983, tablet_31981]
  for i, tablet in enumerate(old_tablets):
    tablet.init_tablet('master', 'test_keyspace', str(i))
    utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
  utils.validate_topology()

  for i, tablet in enumerate(old_tablets):
    tablet.populate(
      "vt_test_keyspace",
      [create_template % table for table in tables],
      sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
    tablet.start_vttablet()
    utils.run_vtctl('MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)

  utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
  tablet_62344.init_tablet('master', 'test_keyspace_new', '-0000000000000028', dbname='not_vt_test_keyspace')
  utils.run_vtctl('RebuildShardGraph test_keyspace_new/-0000000000000028')
  utils.validate_topology()
  tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
  tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created

  # 0x28 = 40
  source_aliases = ' '.join(t.tablet_alias for t in old_tablets)
  utils.run_vtctl('MultiRestore %s %s' % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True)
  time.sleep(1)
  for table in tables:
    rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
    if len(rows) == 0:
      raise utils.TestError("There are no rows in the restored database.")
    for row in rows:
      if row[0] > 32:
        raise utils.TestError("Bad row: %s" % row)
  for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
    tablet.kill_vttablet()
Beispiel #57
0
    def test_reparent_doesnt_hang_if_master_fails(self):
        """Makes sure a failed master populate doesn't hang."""
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        # create the database so vttablets start, as they are serving
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62044.create_db('vt_test_keyspace')
        tablet_41983.create_db('vt_test_keyspace')
        tablet_31981.create_db('vt_test_keyspace')

        # Start up vttablet
        for t in [tablet_62344, tablet_62044, tablet_31981, tablet_41983]:
            t.init_tablet('replica',
                          'test_keyspace',
                          '0',
                          start=True,
                          wait_for_start=False)

        # wait for all tablets to start
        for t in [tablet_62344, tablet_62044, tablet_31981, tablet_41983]:
            t.wait_for_vttablet_state('NOT_SERVING')

        # Force the slaves to reparent. Will create the _vt.reparent_journal table.
        utils.run_vtctl([
            'InitShardMaster', '-force', 'test_keyspace/0',
            tablet_62344.tablet_alias
        ])
        utils.validate_topology(ping_tablets=True)

        # Change the schema of the _vt.reparent_journal table, so that
        # inserts into it will fail. That will make the master fail.
        tablet_62344.mquery(
            '_vt', 'ALTER TABLE reparent_journal'
            ' DROP COLUMN replication_position')

        # Perform a planned reparent operation, the master will fail the
        # insert.  The slaves should then abort right away. If this fails,
        # the test will timeout.
        _, stderr = utils.run_vtctl([
            '-wait-time', '3600s', 'PlannedReparentShard', '-keyspace_shard',
            'test_keyspace/0', '-new_master', tablet_62044.tablet_alias
        ],
                                    expect_fail=True)
        self.assertIn(
            'master failed to PopulateReparentJournal, canceling slaves',
            stderr)

        # Clean up the tablets.
        tablet.kill_tablets(
            [tablet_62344, tablet_62044, tablet_41983, tablet_31981])