def test_multisnapshot_and_restore_vtctl():
tables = ["vt_insert_test", "vt_insert_test1"]
create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB"""
insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl("CreateKeyspace -force test_keyspace")
# Start three tablets for three different shards. At this point the
# sharding schema is not really important, as long as it is
# consistent.
new_spec = "-0000000000000028-"
old_tablets = [tablet_62044, tablet_41983, tablet_31981]
for i, tablet in enumerate(old_tablets):
tablet.init_tablet("master", "test_keyspace", str(i))
utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
utils.validate_topology()
for i, tablet in enumerate(old_tablets):
tablet.populate(
"vt_test_keyspace",
[create_template % table for table in tables],
sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
)
tablet.start_vttablet()
utils.run_vtctl(
"MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id" % (new_spec, tablet.tablet_alias),
trap_output=True,
)
utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
tablet_62344.init_tablet("master", "test_keyspace_new", "-0000000000000028", dbname="not_vt_test_keyspace")
utils.run_vtctl("RebuildShardGraph test_keyspace_new/-0000000000000028")
utils.validate_topology()
tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
tablet_62344.start_vttablet(wait_for_state="CONNECTING") # db not created
# 0x28 = 40
source_aliases = " ".join(t.tablet_alias for t in old_tablets)
utils.run_vtctl(
"MultiRestore %s %s" % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True
)
time.sleep(1)
for table in tables:
rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
if len(rows) == 0:
raise utils.TestError("There are no rows in the restored database.")
for row in rows:
if row[0] > 32:
raise utils.TestError("Bad row: %s" % row)
for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
tablet.kill_vttablet()
def run_shard_tablets(self, shard_name, shard_tablets, create_db=True, create_table=True, wait_state='SERVING'):
"""Handles all the necessary work for initially running a shard's tablets.
This encompasses the following steps:
1. InitTablet for the appropriate tablets and types
2. (optional) Create db
3. Starting vttablets
4. Waiting for the appropriate vttablet state
5. Force reparent to the master tablet
6. RebuildKeyspaceGraph
7. (optional) Running initial schema setup
Args:
shard_name - the name of the shard to start tablets in
shard_tablets - an instance of ShardTablets for the given shard
wait_state - string, the vttablet state that we should wait for
create_db - boolean, True iff we should create a db on the tablets
create_table - boolean, True iff we should create a table on the tablets
"""
shard_tablets.master.init_tablet('master', 'test_keyspace', shard_name)
for tablet in shard_tablets.replicas:
tablet.init_tablet('replica', 'test_keyspace', shard_name)
for tablet in shard_tablets.rdonlys:
tablet.init_tablet('rdonly', 'test_keyspace', shard_name)
# Start tablets (and possibly create databases)
for tablet in shard_tablets.all_tablets:
if create_db:
tablet.create_db('vt_test_keyspace')
tablet.start_vttablet(wait_for_state=None)
# Wait for tablet state to change after starting all tablets. This allows
# us to start all tablets at once, instead of sequentially waiting.
for tablet in shard_tablets.all_tablets:
tablet.wait_for_vttablet_state(wait_state)
# Reparent to choose an initial master
utils.run_vtctl(['InitShardMaster', 'test_keyspace/%s' % shard_name,
shard_tablets.master.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
create_table_sql = (
'create table worker_test('
'id bigint unsigned,'
'msg varchar(64),'
'keyspace_id bigint(20) unsigned not null,'
'primary key (id),'
'index by_msg (msg)'
') Engine=InnoDB'
)
if create_table:
utils.run_vtctl(['ApplySchema',
'-sql=' + create_table_sql,
'test_keyspace'],
auto_log=True)
def test_multisnapshot_and_restore_vtctl():
tables = ['vt_insert_test', 'vt_insert_test1']
create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id)
) Engine=InnoDB'''
insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force test_keyspace')
# Start three tablets for three different shards. At this point the
# sharding schema is not really important, as long as it is
# consistent.
new_spec = '-0000000000000028-'
old_tablets = [tablet_62044, tablet_41983, tablet_31981]
for i, tablet in enumerate(old_tablets):
tablet.init_tablet('master', 'test_keyspace', str(i))
utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
utils.validate_topology()
for i, tablet in enumerate(old_tablets):
tablet.populate(
"vt_test_keyspace",
[create_template % table for table in tables],
sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
tablet.start_vttablet()
utils.run_vtctl('MultiSnapshot -force -maximum-file-size=1 -spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)
utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
tablet_62344.init_tablet('master', 'test_keyspace_new', '-0000000000000028', dbname='not_vt_test_keyspace')
utils.run_vtctl('RebuildShardGraph test_keyspace_new/-0000000000000028')
utils.validate_topology()
tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created
# 0x28 = 40
source_aliases = ' '.join(t.tablet_alias for t in old_tablets)
utils.run_vtctl('MultiRestore %s %s' % (tablet_62344.tablet_alias, source_aliases), auto_log=True, raise_on_error=True)
time.sleep(1)
for table in tables:
rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
if len(rows) == 0:
raise utils.TestError("There are no rows in the restored database.")
for row in rows:
if row[0] > 32:
raise utils.TestError("Bad row: %s" % row)
for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
tablet.kill_vttablet()
def run_shard_tablets(self, shard_name, shard_tablets, create_db=True, create_table=True, wait_state='SERVING'):
"""Handles all the necessary work for initially running a shard's tablets.
This encompasses the following steps:
1. (optional) Create db
2. Starting vttablets and let themselves init them
3. Waiting for the appropriate vttablet state
4. Force reparent to the master tablet
5. RebuildKeyspaceGraph
7. (optional) Running initial schema setup
Args:
shard_name - the name of the shard to start tablets in
shard_tablets - an instance of ShardTablets for the given shard
wait_state - string, the vttablet state that we should wait for
create_db - boolean, True iff we should create a db on the tablets
create_table - boolean, True iff we should create a table on the tablets
"""
# If requested, create databases.
for tablet in shard_tablets.all_tablets:
if create_db:
tablet.create_db('vt_test_keyspace')
# Start tablets.
# Specifying "target_tablet_type" enables the health check i.e. tablets will
# be automatically returned to the serving graph after a SplitClone or SplitDiff.
# NOTE: The future master has to be started with type "replica".
shard_tablets.master.start_vttablet(wait_for_state=None, target_tablet_type='replica',
init_keyspace='test_keyspace', init_shard=shard_name)
for tablet in shard_tablets.replicas:
tablet.start_vttablet(wait_for_state=None, target_tablet_type='replica',
init_keyspace='test_keyspace', init_shard=shard_name)
for tablet in shard_tablets.rdonlys:
tablet.start_vttablet(wait_for_state=None, target_tablet_type='rdonly',
init_keyspace='test_keyspace', init_shard=shard_name)
# Block until tablets are up and we can enable replication.
# We don't care about the tablets' state which may have been changed by the
# health check from SERVING to NOT_SERVING anyway.
for tablet in shard_tablets.all_tablets:
tablet.wait_for_vttablet_state('NOT_SERVING')
# Reparent to choose an initial master and enable replication.
utils.run_vtctl(['InitShardMaster', '-force', 'test_keyspace/%s' % shard_name,
shard_tablets.master.tablet_alias], auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# Enforce a health check instead of waiting for the next periodic one.
# (saves up to 1 second execution time on average)
if wait_state == 'SERVING':
for tablet in shard_tablets.replicas:
utils.run_vtctl(["RunHealthCheck", tablet.tablet_alias, "replica"])
for tablet in shard_tablets.rdonlys:
utils.run_vtctl(["RunHealthCheck", tablet.tablet_alias, "rdonly"])
# Wait for tablet state to change after starting all tablets. This allows
# us to start all tablets at once, instead of sequentially waiting.
# NOTE: Replication has to be enabled first or the health check will
# set a a replica or rdonly tablet back to NOT_SERVING.
for tablet in shard_tablets.all_tablets:
tablet.wait_for_vttablet_state(wait_state)
create_table_sql = (
'create table worker_test('
'id bigint unsigned,'
'msg varchar(64),'
'keyspace_id bigint(20) unsigned not null,'
'primary key (id),'
'index by_msg (msg)'
') Engine=InnoDB'
)
if create_table:
utils.run_vtctl(['ApplySchema',
'-sql=' + create_table_sql,
'test_keyspace'],
auto_log=True)
def test_multisnapshot_and_restore():
tables = ['vt_insert_test', 'vt_insert_test1']
create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB'''
create_view = '''create view vt_insert_view(id, msg) as select id, msg from vt_insert_test'''
insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force test_keyspace')
# Start three tablets for three different shards. At this point the
# sharding schema is not really important, as long as it is
# consistent.
new_spec = '-0000000000000028-'
old_tablets = [tablet_62044, tablet_41983, tablet_31981]
for i, tablet in enumerate(old_tablets):
tablet.init_tablet('master', 'test_keyspace', str(i))
utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
utils.validate_topology()
for i, tablet in enumerate(old_tablets):
tablet.populate(
"vt_test_keyspace",
[create_template % table for table in tables] + [create_view],
sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
tablet.start_vttablet()
utils.run_vtctl('MultiSnapshot --force --spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)
utils.pause("After snapshot")
# try to get the schema on the source, make sure the view is there
out, err = utils.run_vtctl('GetSchema --include-views ' +
tablet_62044.tablet_alias,
log_level='INFO', trap_output=True)
if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
raise utils.TestError('Unexpected GetSchema --include-views output: %s' % err)
out, err = utils.run_vtctl('GetSchema ' +
tablet_62044.tablet_alias,
log_level='INFO', trap_output=True)
if 'vt_insert_view' in err:
raise utils.TestError('Unexpected GetSchema output: %s' % err)
utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
tablet_62344.init_tablet('master', 'test_keyspace_new', "0", dbname="not_vt_test_keyspace")
utils.run_vtctl('RebuildShardGraph test_keyspace_new/0')
utils.validate_topology()
tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created
tablet_urls = ' '.join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)
# 0x28 = 40
err = tablet_62344.mysqlctl("multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam,populateBlpRecovery(6514) not_vt_test_keyspace %s" % tablet_urls).wait()
if err != 0:
raise utils.TestError("mysqlctl failed: %u" % err)
for table in tables:
rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
if len(rows) == 0:
raise utils.TestError("There are no rows in the restored database.")
for row in rows:
if row[0] > 32:
raise utils.TestError("Bad row: %s" % row)
rows = tablet_62344.mquery('_vt', 'select * from blp_checkpoint')
if len(rows) != 3:
raise utils.TestError("Was expecting 3 rows in blp_checkpoint but got: %s" % str(rows))
# try to get the schema on multi-restored guy, make sure the view is there
out, err = utils.run_vtctl('GetSchema --include-views ' +
tablet_62344.tablet_alias,
log_level='INFO', trap_output=True)
if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
raise utils.TestError('Unexpected GetSchema --include-views output after multirestore: %s' % err)
for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
tablet.kill_vttablet()
def run_shard_tablets(self,
shard_name,
shard_tablets,
create_db=True,
create_table=True,
wait_state='SERVING'):
"""Handles all the necessary work for initially running a shard's tablets.
This encompasses the following steps:
1. (optional) Create db
2. Starting vttablets and let themselves init them
3. Waiting for the appropriate vttablet state
4. Force reparent to the master tablet
5. RebuildKeyspaceGraph
7. (optional) Running initial schema setup
Args:
shard_name: the name of the shard to start tablets in
shard_tablets: an instance of ShardTablets for the given shard
create_db: boolean, True iff we should create a db on the tablets
create_table: boolean, True iff we should create a table on the tablets
wait_state: string, the vttablet state that we should wait for
"""
# If requested, create databases.
for tablet in shard_tablets.all_tablets:
if create_db:
tablet.create_db('vt_test_keyspace')
# Start tablets.
#
# Specifying 'target_tablet_type' enables the health check
# i.e. tablets will be automatically returned to the serving graph
# after a SplitClone or SplitDiff.
#
# NOTE: The future master has to be started with type 'replica'.
shard_tablets.master.start_vttablet(wait_for_state=None,
target_tablet_type='replica',
init_keyspace='test_keyspace',
init_shard=shard_name)
for tablet in shard_tablets.replicas:
tablet.start_vttablet(wait_for_state=None,
target_tablet_type='replica',
init_keyspace='test_keyspace',
init_shard=shard_name)
for tablet in shard_tablets.rdonlys:
tablet.start_vttablet(wait_for_state=None,
target_tablet_type='rdonly',
init_keyspace='test_keyspace',
init_shard=shard_name)
# Block until tablets are up and we can enable replication.
# We don't care about the tablets' state which may have been changed by the
# health check from SERVING to NOT_SERVING anyway.
for tablet in shard_tablets.all_tablets:
tablet.wait_for_vttablet_state('NOT_SERVING')
# Reparent to choose an initial master and enable replication.
utils.run_vtctl([
'InitShardMaster', '-force',
'test_keyspace/%s' % shard_name, shard_tablets.master.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
# Enforce a health check instead of waiting for the next periodic one.
# (saves up to 1 second execution time on average)
if wait_state == 'SERVING':
for tablet in shard_tablets.replicas:
utils.run_vtctl(
['RunHealthCheck', tablet.tablet_alias, 'replica'])
for tablet in shard_tablets.rdonlys:
utils.run_vtctl(
['RunHealthCheck', tablet.tablet_alias, 'rdonly'])
# Wait for tablet state to change after starting all tablets. This allows
# us to start all tablets at once, instead of sequentially waiting.
# NOTE: Replication has to be enabled first or the health check will
# set a a replica or rdonly tablet back to NOT_SERVING.
for tablet in shard_tablets.all_tablets:
tablet.wait_for_vttablet_state(wait_state)
create_table_sql = ('create table worker_test('
'id bigint unsigned,'
'msg varchar(64),'
'keyspace_id bigint(20) unsigned not null,'
'primary key (id),'
'index by_msg (msg)'
') Engine=InnoDB')
if create_table:
utils.run_vtctl(
['ApplySchema', '-sql=' + create_table_sql, 'test_keyspace'],
auto_log=True)
def test_multisnapshot_and_restore():
tables = ["vt_insert_test", "vt_insert_test1"]
create_template = """create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB"""
create_view = """create view vt_insert_view(id, msg) as select id, msg from vt_insert_test"""
insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl("CreateKeyspace -force test_keyspace")
# Start three tablets for three different shards. At this point the
# sharding schema is not really important, as long as it is
# consistent.
new_spec = "-0000000000000028-"
old_tablets = [tablet_62044, tablet_41983, tablet_31981]
for i, tablet in enumerate(old_tablets):
tablet.init_tablet("master", "test_keyspace", str(i))
utils.run_vtctl("RebuildShardGraph test_keyspace/%s" % i)
utils.validate_topology()
for i, tablet in enumerate(old_tablets):
tablet.populate(
"vt_test_keyspace",
[create_template % table for table in tables] + [create_view],
sum([[insert_template % (table, 10 * j + i, 10 * j + i) for j in range(1, 8)] for table in tables], []),
)
tablet.start_vttablet()
utils.run_vtctl("MultiSnapshot --force --spec=%s %s id" % (new_spec, tablet.tablet_alias), trap_output=True)
utils.pause("After snapshot")
# try to get the schema on the source, make sure the view is there
out, err = utils.run_vtctl("GetSchema --include-views " + tablet_62044.tablet_alias, trap_output=True)
if "vt_insert_view" not in err or "VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select" not in err:
raise utils.TestError("Unexpected GetSchema --include-views output: %s" % err)
out, err = utils.run_vtctl("GetSchema " + tablet_62044.tablet_alias, trap_output=True)
if "vt_insert_view" in err:
raise utils.TestError("Unexpected GetSchema output: %s" % err)
utils.run_vtctl("CreateKeyspace -force test_keyspace_new")
tablet_62344.init_tablet("master", "test_keyspace_new", "0", dbname="not_vt_test_keyspace")
utils.run_vtctl("RebuildShardGraph test_keyspace_new/0")
utils.validate_topology()
tablet_62344.mquery("", "DROP DATABASE IF EXISTS not_vt_test_keyspace")
tablet_62344.start_vttablet(wait_for_state="CONNECTING") # db not created
tablet_urls = " ".join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)
# 0x28 = 40
err = tablet_62344.mysqlctl(
"multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam not_vt_test_keyspace %s"
% tablet_urls
).wait()
if err != 0:
raise utils.TestError("mysqlctl failed: %u" % err)
for table in tables:
rows = tablet_62344.mquery("not_vt_test_keyspace", "select id from %s" % table)
if len(rows) == 0:
raise utils.TestError("There are no rows in the restored database.")
for row in rows:
if row[0] > 32:
raise utils.TestError("Bad row: %s" % row)
# try to get the schema on multi-restored guy, make sure the view is there
out, err = utils.run_vtctl("GetSchema --include-views " + tablet_62344.tablet_alias, trap_output=True)
if "vt_insert_view" not in err or "VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select" not in err:
raise utils.TestError("Unexpected GetSchema --include-views output after multirestore: %s" % err)
for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
tablet.kill_vttablet()
def run_shard_tablets(self,
shard_name,
shard_tablets,
create_db=True,
create_table=True,
wait_state='SERVING'):
"""Handles all the necessary work for initially running a shard's tablets.
This encompasses the following steps:
1. InitTablet for the appropriate tablets and types
2. (optional) Create db
3. Starting vttablets
4. Waiting for the appropriate vttablet state
5. Force reparent to the master tablet
6. RebuildKeyspaceGraph
7. (optional) Running initial schema setup
Args:
shard_name - the name of the shard to start tablets in
shard_tablets - an instance of ShardTablets for the given shard
wait_state - string, the vttablet state that we should wait for
create_db - boolean, True iff we should create a db on the tablets
create_table - boolean, True iff we should create a table on the tablets
"""
shard_tablets.master.init_tablet('master', 'test_keyspace', shard_name)
for tablet in shard_tablets.replicas:
tablet.init_tablet('replica', 'test_keyspace', shard_name)
for tablet in shard_tablets.rdonlys:
tablet.init_tablet('rdonly', 'test_keyspace', shard_name)
# Start tablets (and possibly create databases)
for tablet in shard_tablets.all_tablets:
if create_db:
tablet.create_db('vt_test_keyspace')
tablet.start_vttablet(wait_for_state=None)
# Wait for tablet state to change after starting all tablets. This allows
# us to start all tablets at once, instead of sequentially waiting.
for tablet in shard_tablets.all_tablets:
tablet.wait_for_vttablet_state(wait_state)
# Reparent to choose an initial master
utils.run_vtctl([
'InitShardMaster',
'test_keyspace/%s' % shard_name, shard_tablets.master.tablet_alias
],
auto_log=True)
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
auto_log=True)
create_table_sql = ('create table worker_test('
'id bigint unsigned,'
'msg varchar(64),'
'keyspace_id bigint(20) unsigned not null,'
'primary key (id),'
'index by_msg (msg)'
') Engine=InnoDB')
if create_table:
utils.run_vtctl(
['ApplySchema', '-sql=' + create_table_sql, 'test_keyspace'],
auto_log=True)
