Exemple #1
0
    def tearDown(self):
        """Does the minimum to reset topology and tablets to their initial states.

    When benchmarked, this seemed to take around 30% of the time of
    (setupModule + tearDownModule).

    FIXME(aaijazi): doing this in parallel greatly reduces the time it takes.
    See the kill_tablets method in tablet.py.
    """

        utils.run_vtctl(['ListAllTablets', 'test_nj'])

        for shard_tablet in [
                all_shard_tablets, shard_0_tablets, shard_1_tablets
        ]:
            for tablet in shard_tablet.all_tablets:
                tablet.reset_replication()
                tablet.clean_dbs()
                tablet.kill_vttablet()
                # we allow failures here as some tablets will be gone sometimes
                # (the master tablets after an emergency reparent)
                utils.run_vtctl(
                    ['DeleteTablet', '-allow_master', tablet.tablet_alias],
                    auto_log=True,
                    raise_on_error=False)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                        auto_log=True)
        for shard in ['0', '-80', '80-']:
            utils.run_vtctl(
                ['DeleteShard', 'test_keyspace/%s' % shard], auto_log=True)
Exemple #2
0
  def tearDown(self):
    """Does the minimum to reset topology and tablets to their initial states.

    When benchmarked, this seemed to take around 30% of the time of
    (setupModule + tearDownModule).

    FIXME(aaijazi): doing this in parallel greatly reduces the time it takes.
    See the kill_tablets method in tablet.py.
    """

    utils.run_vtctl(['ListAllTablets', 'test_nj'])
    
    for shard_tablet in [all_shard_tablets, shard_0_tablets, shard_1_tablets]:
      for tablet in shard_tablet.all_tablets:
        tablet.reset_replication()
        tablet.clean_dbs()
        tablet.kill_vttablet()
        # we allow failures here as some tablets will be gone sometimes
        # (the master tablets after an emergency reparent)
        utils.run_vtctl(['DeleteTablet', '-allow_master', tablet.tablet_alias],
                        auto_log=True, raise_on_error=False)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
    for shard in ['0', '-80', '80-']:
      utils.run_vtctl(
          ['DeleteShard', 'test_keyspace/%s' % shard], auto_log=True)
def test_multisnapshot_and_restore_vtctl():
    tables = ["vt_insert_test", "vt_insert_test1"]
    create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB"""
    insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # Start three tablets for three different shards. At this point the
    # sharding schema is not really important, as long as it is
    # consistent.
    new_spec = "-0000000000000028-"
    old_tablets = [tablet_62044, tablet_41983, tablet_31981]
    for i, tablet in enumerate(old_tablets):
        tablet.init_tablet("master", "test_keyspace", str(i))
        utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
    utils.validate_topology()

    for i, tablet in enumerate(old_tablets):
        tablet.populate(
            "vt_test_keyspace",
            [create_template % table for table in tables],
            sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
        )
        tablet.start_vttablet()
        utils.run_vtctl(
            "MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id" % (new_spec, tablet.tablet_alias),
            trap_output=True,
        )

    utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
    tablet_62344.init_tablet("master", "test_keyspace_new", "-0000000000000028", dbname="not_vt_test_keyspace")
    utils.run_vtctl("RebuildShardGraph test_keyspace_new/-0000000000000028")
    utils.validate_topology()
    tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
    tablet_62344.start_vttablet(wait_for_state="CONNECTING")  # db not created

    # 0x28 = 40
    source_aliases = " ".join(t.tablet_alias for t in old_tablets)
    utils.run_vtctl(
        "MultiRestore %s %s" % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True
    )
    time.sleep(1)
    for table in tables:
        rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
        if len(rows) == 0:
            raise utils.TestError("There are no rows in the restored database.")
        for row in rows:
            if row[0] > 32:
                raise utils.TestError("Bad row: %s" % row)
    for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
        tablet.kill_vttablet()
Exemple #4
0
def test_multisnapshot_and_restore_vtctl():
  tables = ['vt_insert_test', 'vt_insert_test1']
  create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB'''
  insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  # Start three tablets for three different shards. At this point the
  # sharding schema is not really important, as long as it is
  # consistent.
  new_spec = '-0000000000000028-'
  old_tablets = [tablet_62044, tablet_41983, tablet_31981]
  for i, tablet in enumerate(old_tablets):
    tablet.init_tablet('master', 'test_keyspace', str(i))
    utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
  utils.validate_topology()

  for i, tablet in enumerate(old_tablets):
    tablet.populate(
      "vt_test_keyspace",
      [create_template % table for table in tables],
      sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
    tablet.start_vttablet()
    utils.run_vtctl('MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)

  utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
  tablet_62344.init_tablet('master', 'test_keyspace_new', '-0000000000000028', dbname='not_vt_test_keyspace')
  utils.run_vtctl('RebuildShardGraph test_keyspace_new/-0000000000000028')
  utils.validate_topology()
  tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
  tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created

  # 0x28 = 40
  source_aliases = ' '.join(t.tablet_alias for t in old_tablets)
  utils.run_vtctl('MultiRestore %s %s' % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True)
  time.sleep(1)
  for table in tables:
    rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
    if len(rows) == 0:
      raise utils.TestError("There are no rows in the restored database.")
    for row in rows:
      if row[0] > 32:
        raise utils.TestError("Bad row: %s" % row)
  for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
    tablet.kill_vttablet()
Exemple #5
0
  def tearDown(self):
    """Tries to do the minimum to reset topology and tablets to their initial states.

    When benchmarked, this seemed to take around 30% of the time of (setupModule +
    tearDownModule).
    """
    for shard_tablet in [shard_tablets, shard_0_tablets, shard_1_tablets]:
      for tablet in shard_tablet.all_tablets:
        tablet.clean_dbs()
        tablet.scrap(force=True, skip_rebuild=True)
        utils.run_vtctl(['DeleteTablet', tablet.tablet_alias], auto_log=True)
        tablet.kill_vttablet()
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
    for shard in ['0', '-80', '80-']:
      utils.run_vtctl(['DeleteShard', 'test_keyspace/%s' % shard], auto_log=True)
Exemple #6
0
    def tearDown(self):
        """Does the minimum to reset topology and tablets to their initial states.

    When benchmarked, this seemed to take around 30% of the time of
    (setupModule + tearDownModule).
    """
        for shard_tablet in [all_shard_tablets, shard_0_tablets, shard_1_tablets]:
            for tablet in shard_tablet.all_tablets:
                tablet.reset_replication()
                tablet.clean_dbs()
                tablet.scrap(force=True, skip_rebuild=True)
                utils.run_vtctl(["DeleteTablet", tablet.tablet_alias], auto_log=True)
                tablet.kill_vttablet()
        utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)
        for shard in ["0", "-80", "80-"]:
            utils.run_vtctl(["DeleteShard", "test_keyspace/%s" % shard], auto_log=True)
Exemple #7
0
  def tearDown(self):
    """Does the minimum to reset topology and tablets to their initial states.

    When benchmarked, this seemed to take around 30% of the time of
    (setupModule + tearDownModule).
    """
    for shard_tablet in [shard_tablets, shard_0_tablets, shard_1_tablets]:
      for tablet in shard_tablet.all_tablets:
        tablet.reset_replication()
        tablet.clean_dbs()
        tablet.scrap(force=True, skip_rebuild=True)
        utils.run_vtctl(['DeleteTablet', tablet.tablet_alias], auto_log=True)
        tablet.kill_vttablet()
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
    for shard in ['0', '-80', '80-']:
      utils.run_vtctl(
          ['DeleteShard', 'test_keyspace/%s' % shard], auto_log=True)
Exemple #8
0
def test_multisnapshot_and_restore():
  tables = ['vt_insert_test', 'vt_insert_test1']
  create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB'''
  create_view = '''create view vt_insert_view(id, msg) as select id, msg from vt_insert_test'''
  insert_template = "insert into %s (id, msg) values (%s, 'test %s')"

  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  # Start three tablets for three different shards. At this point the
  # sharding schema is not really important, as long as it is
  # consistent.
  new_spec = '-0000000000000028-'
  old_tablets = [tablet_62044, tablet_41983, tablet_31981]
  for i, tablet in enumerate(old_tablets):
    tablet.init_tablet('master', 'test_keyspace', str(i))
    utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
  utils.validate_topology()

  for i, tablet in enumerate(old_tablets):
    tablet.populate(
      "vt_test_keyspace",
      [create_template % table for table in tables] + [create_view],
      sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
    tablet.start_vttablet()
    utils.run_vtctl('MultiSnapshot --force  --spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)

  utils.pause("After snapshot")

  # try to get the schema on the source, make sure the view is there
  out, err = utils.run_vtctl('GetSchema --include-views ' +
                             tablet_62044.tablet_alias,
                             log_level='INFO', trap_output=True)
  if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
    raise utils.TestError('Unexpected GetSchema --include-views output: %s' % err)
  out, err = utils.run_vtctl('GetSchema ' +
                             tablet_62044.tablet_alias,
                             log_level='INFO', trap_output=True)
  if 'vt_insert_view' in err:
    raise utils.TestError('Unexpected GetSchema output: %s' % err)

  utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
  tablet_62344.init_tablet('master', 'test_keyspace_new', "0", dbname="not_vt_test_keyspace")
  utils.run_vtctl('RebuildShardGraph test_keyspace_new/0')
  utils.validate_topology()
  tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
  tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created

  tablet_urls = ' '.join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)

  # 0x28 = 40
  err = tablet_62344.mysqlctl("multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam,populateBlpRecovery(6514) not_vt_test_keyspace %s" % tablet_urls).wait()
  if err != 0:
    raise utils.TestError("mysqlctl failed: %u" % err)
  for table in tables:
    rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
    if len(rows) == 0:
      raise utils.TestError("There are no rows in the restored database.")
    for row in rows:
      if row[0] > 32:
        raise utils.TestError("Bad row: %s" % row)
  rows = tablet_62344.mquery('_vt', 'select * from blp_checkpoint')
  if len(rows) != 3:
    raise utils.TestError("Was expecting 3 rows in blp_checkpoint but got: %s" % str(rows))

  # try to get the schema on multi-restored guy, make sure the view is there
  out, err = utils.run_vtctl('GetSchema --include-views ' +
                             tablet_62344.tablet_alias,
                             log_level='INFO', trap_output=True)
  if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
    raise utils.TestError('Unexpected GetSchema --include-views output after multirestore: %s' % err)

  for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
    tablet.kill_vttablet()
def test_multisnapshot_and_restore():
    tables = ["vt_insert_test", "vt_insert_test1"]
    create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB"""
    create_view = """create view vt_insert_view(id, msg) as select id, msg from vt_insert_test"""
    insert_template = "insert into %s (id, msg) values (%s, 'test %s')"

    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # Start three tablets for three different shards. At this point the
    # sharding schema is not really important, as long as it is
    # consistent.
    new_spec = "-0000000000000028-"
    old_tablets = [tablet_62044, tablet_41983, tablet_31981]
    for i, tablet in enumerate(old_tablets):
        tablet.init_tablet("master", "test_keyspace", str(i))
        utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
    utils.validate_topology()

    for i, tablet in enumerate(old_tablets):
        tablet.populate(
            "vt_test_keyspace",
            [create_template % table for table in tables] + [create_view],
            sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
        )
        tablet.start_vttablet()
        utils.run_vtctl("MultiSnapshot --force  --spec=%s %s id" % (new_spec, tablet.tablet_alias), trap_output=True)

    utils.pause("After snapshot")

    # try to get the schema on the source, make sure the view is there
    out, err = utils.run_vtctl("GetSchema --include-views " + tablet_62044.tablet_alias, trap_output=True)
    if "vt_insert_view" not in err or "VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select" not in err:
        raise utils.TestError("Unexpected GetSchema --include-views output: %s" % err)
    out, err = utils.run_vtctl("GetSchema " + tablet_62044.tablet_alias, trap_output=True)
    if "vt_insert_view" in err:
        raise utils.TestError("Unexpected GetSchema output: %s" % err)

    utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
    tablet_62344.init_tablet("master", "test_keyspace_new", "0", dbname="not_vt_test_keyspace")
    utils.run_vtctl("RebuildShardGraph test_keyspace_new/0")
    utils.validate_topology()
    tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
    tablet_62344.start_vttablet(wait_for_state="CONNECTING")  # db not created

    tablet_urls = " ".join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)

    # 0x28 = 40
    err = tablet_62344.mysqlctl(
        "multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam not_vt_test_keyspace %s"
        % tablet_urls
    ).wait()
    if err != 0:
        raise utils.TestError("mysqlctl failed: %u" % err)
    for table in tables:
        rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
        if len(rows) == 0:
            raise utils.TestError("There are no rows in the restored database.")
        for row in rows:
            if row[0] > 32:
                raise utils.TestError("Bad row: %s" % row)

    # try to get the schema on multi-restored guy, make sure the view is there
    out, err = utils.run_vtctl("GetSchema --include-views " + tablet_62344.tablet_alias, trap_output=True)
    if "vt_insert_view" not in err or "VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select" not in err:
        raise utils.TestError("Unexpected GetSchema --include-views output after multirestore: %s" % err)

    for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
        tablet.kill_vttablet()