コード例 #1
0
def test_multisnapshot_and_restore_vtctl():
    tables = ["vt_insert_test", "vt_insert_test1"]
    create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB"""
    insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # Start three tablets for three different shards. At this point the
    # sharding schema is not really important, as long as it is
    # consistent.
    new_spec = "-0000000000000028-"
    old_tablets = [tablet_62044, tablet_41983, tablet_31981]
    for i, tablet in enumerate(old_tablets):
        tablet.init_tablet("master", "test_keyspace", str(i))
        utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
    utils.validate_topology()

    for i, tablet in enumerate(old_tablets):
        tablet.populate(
            "vt_test_keyspace",
            [create_template % table for table in tables],
            sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
        )
        tablet.start_vttablet()
        utils.run_vtctl(
            "MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id" % (new_spec, tablet.tablet_alias),
            trap_output=True,
        )

    utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
    tablet_62344.init_tablet("master", "test_keyspace_new", "-0000000000000028", dbname="not_vt_test_keyspace")
    utils.run_vtctl("RebuildShardGraph test_keyspace_new/-0000000000000028")
    utils.validate_topology()
    tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
    tablet_62344.start_vttablet(wait_for_state="CONNECTING")  # db not created

    # 0x28 = 40
    source_aliases = " ".join(t.tablet_alias for t in old_tablets)
    utils.run_vtctl(
        "MultiRestore %s %s" % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True
    )
    time.sleep(1)
    for table in tables:
        rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
        if len(rows) == 0:
            raise utils.TestError("There are no rows in the restored database.")
        for row in rows:
            if row[0] > 32:
                raise utils.TestError("Bad row: %s" % row)
    for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
        tablet.kill_vttablet()
コード例 #2
0
ファイル: worker.py プロジェクト: pranjal5215/vitess
  def run_shard_tablets(self, shard_name, shard_tablets, create_db=True, create_table=True, wait_state='SERVING'):
    """Handles all the necessary work for initially running a shard's tablets.

    This encompasses the following steps:
      1. InitTablet for the appropriate tablets and types
      2. (optional) Create db
      3. Starting vttablets
      4. Waiting for the appropriate vttablet state
      5. Force reparent to the master tablet
      6. RebuildKeyspaceGraph
      7. (optional) Running initial schema setup

    Args:
      shard_name - the name of the shard to start tablets in
      shard_tablets - an instance of ShardTablets for the given shard
      wait_state - string, the vttablet state that we should wait for
      create_db - boolean, True iff we should create a db on the tablets
      create_table - boolean, True iff we should create a table on the tablets
    """
    shard_tablets.master.init_tablet('master', 'test_keyspace', shard_name)
    for tablet in shard_tablets.replicas:
      tablet.init_tablet('replica', 'test_keyspace', shard_name)
    for tablet in shard_tablets.rdonlys:
      tablet.init_tablet('rdonly', 'test_keyspace', shard_name)

    # Start tablets (and possibly create databases)
    for tablet in shard_tablets.all_tablets:
      if create_db:
        tablet.create_db('vt_test_keyspace')
      tablet.start_vttablet(wait_for_state=None)

    # Wait for tablet state to change after starting all tablets. This allows
    # us to start all tablets at once, instead of sequentially waiting.
    for tablet in shard_tablets.all_tablets:
      tablet.wait_for_vttablet_state(wait_state)

    # Reparent to choose an initial master
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/%s' % shard_name,
                     shard_tablets.master.tablet_alias], auto_log=True)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    create_table_sql = (
      'create table worker_test('
      'id bigint unsigned,'
      'msg varchar(64),'
      'keyspace_id bigint(20) unsigned not null,'
      'primary key (id),'
      'index by_msg (msg)'
      ') Engine=InnoDB'
    )

    if create_table:
      utils.run_vtctl(['ApplySchema',
                       '-sql=' + create_table_sql,
                       'test_keyspace'],
                      auto_log=True)
コード例 #3
0
ファイル: tabletmanager.py プロジェクト: hub501/go-pack
def test_multisnapshot_and_restore_vtctl():
  tables = ['vt_insert_test', 'vt_insert_test1']
  create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB'''
  insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  # Start three tablets for three different shards. At this point the
  # sharding schema is not really important, as long as it is
  # consistent.
  new_spec = '-0000000000000028-'
  old_tablets = [tablet_62044, tablet_41983, tablet_31981]
  for i, tablet in enumerate(old_tablets):
    tablet.init_tablet('master', 'test_keyspace', str(i))
    utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
  utils.validate_topology()

  for i, tablet in enumerate(old_tablets):
    tablet.populate(
      "vt_test_keyspace",
      [create_template % table for table in tables],
      sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
    tablet.start_vttablet()
    utils.run_vtctl('MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)

  utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
  tablet_62344.init_tablet('master', 'test_keyspace_new', '-0000000000000028', dbname='not_vt_test_keyspace')
  utils.run_vtctl('RebuildShardGraph test_keyspace_new/-0000000000000028')
  utils.validate_topology()
  tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
  tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created

  # 0x28 = 40
  source_aliases = ' '.join(t.tablet_alias for t in old_tablets)
  utils.run_vtctl('MultiRestore %s %s' % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True)
  time.sleep(1)
  for table in tables:
    rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
    if len(rows) == 0:
      raise utils.TestError("There are no rows in the restored database.")
    for row in rows:
      if row[0] > 32:
        raise utils.TestError("Bad row: %s" % row)
  for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
    tablet.kill_vttablet()
コード例 #4
0
ファイル: tabletmanager.py プロジェクト: hub501/go-pack
def test_multisnapshot_and_restore():
  tables = ['vt_insert_test', 'vt_insert_test1']
  create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB'''
  create_view = '''create view vt_insert_view(id, msg) as select id, msg from vt_insert_test'''
  insert_template = "insert into %s (id, msg) values (%s, 'test %s')"

  utils.zk_wipe()

  # Start up a master mysql and vttablet
  utils.run_vtctl('CreateKeyspace -force test_keyspace')

  # Start three tablets for three different shards. At this point the
  # sharding schema is not really important, as long as it is
  # consistent.
  new_spec = '-0000000000000028-'
  old_tablets = [tablet_62044, tablet_41983, tablet_31981]
  for i, tablet in enumerate(old_tablets):
    tablet.init_tablet('master', 'test_keyspace', str(i))
    utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
  utils.validate_topology()

  for i, tablet in enumerate(old_tablets):
    tablet.populate(
      "vt_test_keyspace",
      [create_template % table for table in tables] + [create_view],
      sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
    tablet.start_vttablet()
    utils.run_vtctl('MultiSnapshot --force  --spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)

  utils.pause("After snapshot")

  # try to get the schema on the source, make sure the view is there
  out, err = utils.run_vtctl('GetSchema --include-views ' +
                             tablet_62044.tablet_alias,
                             log_level='INFO', trap_output=True)
  if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
    raise utils.TestError('Unexpected GetSchema --include-views output: %s' % err)
  out, err = utils.run_vtctl('GetSchema ' +
                             tablet_62044.tablet_alias,
                             log_level='INFO', trap_output=True)
  if 'vt_insert_view' in err:
    raise utils.TestError('Unexpected GetSchema output: %s' % err)

  utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
  tablet_62344.init_tablet('master', 'test_keyspace_new', "0", dbname="not_vt_test_keyspace")
  utils.run_vtctl('RebuildShardGraph test_keyspace_new/0')
  utils.validate_topology()
  tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
  tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created

  tablet_urls = ' '.join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)

  # 0x28 = 40
  err = tablet_62344.mysqlctl("multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam,populateBlpRecovery(6514) not_vt_test_keyspace %s" % tablet_urls).wait()
  if err != 0:
    raise utils.TestError("mysqlctl failed: %u" % err)
  for table in tables:
    rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
    if len(rows) == 0:
      raise utils.TestError("There are no rows in the restored database.")
    for row in rows:
      if row[0] > 32:
        raise utils.TestError("Bad row: %s" % row)
  rows = tablet_62344.mquery('_vt', 'select * from blp_checkpoint')
  if len(rows) != 3:
    raise utils.TestError("Was expecting 3 rows in blp_checkpoint but got: %s" % str(rows))

  # try to get the schema on multi-restored guy, make sure the view is there
  out, err = utils.run_vtctl('GetSchema --include-views ' +
                             tablet_62344.tablet_alias,
                             log_level='INFO', trap_output=True)
  if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
    raise utils.TestError('Unexpected GetSchema --include-views output after multirestore: %s' % err)

  for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
    tablet.kill_vttablet()
コード例 #5
0
def test_multisnapshot_and_restore():
    tables = ["vt_insert_test", "vt_insert_test1"]
    create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB"""
    create_view = """create view vt_insert_view(id, msg) as select id, msg from vt_insert_test"""
    insert_template = "insert into %s (id, msg) values (%s, 'test %s')"

    utils.zk_wipe()

    # Start up a master mysql and vttablet
    utils.run_vtctl("CreateKeyspace -force test_keyspace")

    # Start three tablets for three different shards. At this point the
    # sharding schema is not really important, as long as it is
    # consistent.
    new_spec = "-0000000000000028-"
    old_tablets = [tablet_62044, tablet_41983, tablet_31981]
    for i, tablet in enumerate(old_tablets):
        tablet.init_tablet("master", "test_keyspace", str(i))
        utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
    utils.validate_topology()

    for i, tablet in enumerate(old_tablets):
        tablet.populate(
            "vt_test_keyspace",
            [create_template % table for table in tables] + [create_view],
            sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
        )
        tablet.start_vttablet()
        utils.run_vtctl("MultiSnapshot --force  --spec=%s %s id" % (new_spec, tablet.tablet_alias), trap_output=True)

    utils.pause("After snapshot")

    # try to get the schema on the source, make sure the view is there
    out, err = utils.run_vtctl("GetSchema --include-views " + tablet_62044.tablet_alias, trap_output=True)
    if "vt_insert_view" not in err or "VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select" not in err:
        raise utils.TestError("Unexpected GetSchema --include-views output: %s" % err)
    out, err = utils.run_vtctl("GetSchema " + tablet_62044.tablet_alias, trap_output=True)
    if "vt_insert_view" in err:
        raise utils.TestError("Unexpected GetSchema output: %s" % err)

    utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
    tablet_62344.init_tablet("master", "test_keyspace_new", "0", dbname="not_vt_test_keyspace")
    utils.run_vtctl("RebuildShardGraph test_keyspace_new/0")
    utils.validate_topology()
    tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
    tablet_62344.start_vttablet(wait_for_state="CONNECTING")  # db not created

    tablet_urls = " ".join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)

    # 0x28 = 40
    err = tablet_62344.mysqlctl(
        "multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam not_vt_test_keyspace %s"
        % tablet_urls
    ).wait()
    if err != 0:
        raise utils.TestError("mysqlctl failed: %u" % err)
    for table in tables:
        rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
        if len(rows) == 0:
            raise utils.TestError("There are no rows in the restored database.")
        for row in rows:
            if row[0] > 32:
                raise utils.TestError("Bad row: %s" % row)

    # try to get the schema on multi-restored guy, make sure the view is there
    out, err = utils.run_vtctl("GetSchema --include-views " + tablet_62344.tablet_alias, trap_output=True)
    if "vt_insert_view" not in err or "VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select" not in err:
        raise utils.TestError("Unexpected GetSchema --include-views output after multirestore: %s" % err)

    for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
        tablet.kill_vttablet()
コード例 #6
0
ファイル: worker.py プロジェクト: Drooids/vitess
    def run_shard_tablets(self,
                          shard_name,
                          shard_tablets,
                          create_db=True,
                          create_table=True,
                          wait_state='SERVING'):
        """Handles all the necessary work for initially running a shard's tablets.

    This encompasses the following steps:
      1. InitTablet for the appropriate tablets and types
      2. (optional) Create db
      3. Starting vttablets
      4. Waiting for the appropriate vttablet state
      5. Force reparent to the master tablet
      6. RebuildKeyspaceGraph
      7. (optional) Running initial schema setup

    Args:
      shard_name - the name of the shard to start tablets in
      shard_tablets - an instance of ShardTablets for the given shard
      wait_state - string, the vttablet state that we should wait for
      create_db - boolean, True iff we should create a db on the tablets
      create_table - boolean, True iff we should create a table on the tablets
    """
        shard_tablets.master.init_tablet('master', 'test_keyspace', shard_name)
        for tablet in shard_tablets.replicas:
            tablet.init_tablet('replica', 'test_keyspace', shard_name)
        for tablet in shard_tablets.rdonlys:
            tablet.init_tablet('rdonly', 'test_keyspace', shard_name)

        # Start tablets (and possibly create databases)
        for tablet in shard_tablets.all_tablets:
            if create_db:
                tablet.create_db('vt_test_keyspace')
            tablet.start_vttablet(wait_for_state=None)

        # Wait for tablet state to change after starting all tablets. This allows
        # us to start all tablets at once, instead of sequentially waiting.
        for tablet in shard_tablets.all_tablets:
            tablet.wait_for_vttablet_state(wait_state)

        # Reparent to choose an initial master
        utils.run_vtctl([
            'InitShardMaster',
            'test_keyspace/%s' % shard_name, shard_tablets.master.tablet_alias
        ],
                        auto_log=True)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                        auto_log=True)

        create_table_sql = ('create table worker_test('
                            'id bigint unsigned,'
                            'msg varchar(64),'
                            'keyspace_id bigint(20) unsigned not null,'
                            'primary key (id),'
                            'index by_msg (msg)'
                            ') Engine=InnoDB')

        if create_table:
            utils.run_vtctl(
                ['ApplySchema', '-sql=' + create_table_sql, 'test_keyspace'],
                auto_log=True)