def _list_backups(self):
"""Get a list of backup names for the test shard."""
backups, _ = utils.run_vtctl(tablet.get_backup_storage_flags() +
['ListBackups', 'test_keyspace/0'],
mode=utils.VTCTL_VTCTL,
trap_output=True)
return backups.splitlines()
def _remove_backup(self, backup, shard):
"""Remove a named backup from the test shard."""
utils.run_vtctl(tablet.get_backup_storage_flags() +
['RemoveBackup',
'test_keyspace/%s' % shard, backup],
auto_log=True,
mode=utils.VTCTL_VTCTL)
def _restore(self, t, keyspace):
"""Erase mysql/tablet dir, then start tablet with restore enabled."""
self._reset_tablet_dir(t)
# create a recovery keyspace
utils.run_vtctl([
'CreateKeyspace', '-keyspace_type=SNAPSHOT',
'-base_keyspace=test_keyspace', '-snapshot_time',
datetime.utcnow().isoformat("T") + "Z", keyspace
])
# set disable_active_reparents to true and enable_replication_reporter to false
# otherwise replication_reporter will try to restart replication
xtra_args = [
'-disable_active_reparents', '-enable_replication_reporter=false'
]
xtra_args.extend(tablet.get_backup_storage_flags())
if use_xtrabackup:
xtra_args.extend(xtrabackup_args)
t.start_vttablet(wait_for_state='SERVING',
init_tablet_type='replica',
init_keyspace=keyspace,
init_shard='0',
supports_backups=True,
extra_args=xtra_args)
def test_backup(self):
"""test_backup will:
- create a shard with master and replica1 only
- run InitShardMaster
- insert some data
- take a backup
- insert more data on the master
- bring up tablet_replica2 after the fact, let it restore the backup
- check all data is right (before+after backup data)
- list the backup, remove it
"""
for t in tablet_master, tablet_replica1:
t.create_db('vt_test_keyspace')
tablet_master.init_tablet('master', 'test_keyspace', '0', start=True,
supports_backups=True)
tablet_replica1.init_tablet('replica', 'test_keyspace', '0', start=True,
supports_backups=True)
utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
tablet_master.tablet_alias])
# insert data on master, wait for slave to get it
tablet_master.mquery('vt_test_keyspace', self._create_vt_insert_test)
self._insert_master(1)
timeout = 10
while True:
result = tablet_replica1.mquery('vt_test_keyspace', 'select count(*) from vt_insert_test')
if result[0][0] == 1:
break
timeout = utils.wait_step('slave tablet getting data', timeout)
# backup the slave
utils.run_vtctl(['Backup', tablet_replica1.tablet_alias], auto_log=True)
# insert more data on the master
self._insert_master(2)
# now bring up the other slave, health check on, init_tablet on, restore on
tablet_replica2.start_vttablet(wait_for_state='SERVING',
target_tablet_type='replica',
init_keyspace='test_keyspace',
init_shard='0',
supports_backups=True)
# check the new slave has the data
timeout = 10
while True:
result = tablet_replica2.mquery('vt_test_keyspace', 'select count(*) from vt_insert_test')
if result[0][0] == 2:
break
timeout = utils.wait_step('new slave tablet getting data', timeout)
# list the backups
backups, err = utils.run_vtctl(tablet.get_backup_storage_flags() +
['ListBackups', 'test_keyspace/0'],
mode=utils.VTCTL_VTCTL, trap_output=True)
backups = backups.splitlines()
logging.debug("list of backups: %s", backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].startswith(tablet_replica1.tablet_alias))
# remove the backup
utils.run_vtctl(tablet.get_backup_storage_flags() +
['RemoveBackup', 'test_keyspace/0', backups[0]],
auto_log=True, mode=utils.VTCTL_VTCTL)
# make sure the list of backups is empty now
backups, err = utils.run_vtctl(tablet.get_backup_storage_flags() +
['ListBackups', 'test_keyspace/0'],
mode=utils.VTCTL_VTCTL, trap_output=True)
backups = backups.splitlines()
logging.debug("list of backups after remove: %s", backups)
self.assertEqual(len(backups), 0)
for t in tablet_master, tablet_replica1, tablet_replica2:
t.kill_vttablet()
def test_backup(self):
"""Test backup flow.
test_backup will:
- create a shard with master and replica1 only
- run InitShardMaster
- insert some data
- take a backup
- insert more data on the master
- bring up tablet_replica2 after the fact, let it restore the backup
- check all data is right (before+after backup data)
- list the backup, remove it
"""
for t in tablet_master, tablet_replica1:
t.create_db('vt_test_keyspace')
tablet_master.init_tablet('master',
'test_keyspace',
'0',
start=True,
supports_backups=True)
tablet_replica1.init_tablet('replica',
'test_keyspace',
'0',
start=True,
supports_backups=True)
utils.run_vtctl(
['InitShardMaster', 'test_keyspace/0', tablet_master.tablet_alias])
# insert data on master, wait for slave to get it
tablet_master.mquery('vt_test_keyspace', self._create_vt_insert_test)
self._insert_master(1)
timeout = 10
while True:
try:
result = tablet_replica1.mquery(
'vt_test_keyspace', 'select count(*) from vt_insert_test')
if result[0][0] == 1:
break
except MySQLdb.DatabaseError:
# ignore exceptions, we'll just timeout (the tablet creation
# can take some time to replicate, and we get a 'table vt_insert_test
# does not exist exception in some rare cases)
logging.exception('exception waiting for data to replicate')
timeout = utils.wait_step('slave tablet getting data', timeout)
# backup the slave
utils.run_vtctl(['Backup', tablet_replica1.tablet_alias],
auto_log=True)
# insert more data on the master
self._insert_master(2)
# now bring up the other slave, health check on, init_tablet on, restore on
tablet_replica2.start_vttablet(wait_for_state='SERVING',
target_tablet_type='replica',
init_keyspace='test_keyspace',
init_shard='0',
supports_backups=True)
# check the new slave has the data
timeout = 10
while True:
result = tablet_replica2.mquery(
'vt_test_keyspace', 'select count(*) from vt_insert_test')
if result[0][0] == 2:
break
timeout = utils.wait_step('new slave tablet getting data', timeout)
# list the backups
backups, _ = utils.run_vtctl(tablet.get_backup_storage_flags() +
['ListBackups', 'test_keyspace/0'],
mode=utils.VTCTL_VTCTL,
trap_output=True)
backups = backups.splitlines()
logging.debug('list of backups: %s', backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].endswith(tablet_replica1.tablet_alias))
# remove the backup
utils.run_vtctl(tablet.get_backup_storage_flags() +
['RemoveBackup', 'test_keyspace/0', backups[0]],
auto_log=True,
mode=utils.VTCTL_VTCTL)
# make sure the list of backups is empty now
backups, _ = utils.run_vtctl(tablet.get_backup_storage_flags() +
['ListBackups', 'test_keyspace/0'],
mode=utils.VTCTL_VTCTL,
trap_output=True)
backups = backups.splitlines()
logging.debug('list of backups after remove: %s', backups)
self.assertEqual(len(backups), 0)
for t in tablet_master, tablet_replica1, tablet_replica2:
t.kill_vttablet()
def _remove_backup(self, backup):
"""Remove a named backup from the test shard."""
utils.run_vtctl(
tablet.get_backup_storage_flags() +
['RemoveBackup', 'test_keyspace/0', backup],
auto_log=True, mode=utils.VTCTL_VTCTL)
def _list_backups(self):
"""Get a list of backup names for the test shard."""
backups, _ = utils.run_vtctl(tablet.get_backup_storage_flags() +
['ListBackups', 'test_keyspace/0'],
mode=utils.VTCTL_VTCTL, trap_output=True)
return backups.splitlines()
def test_backup(self):
"""Test backup flow.
test_backup will:
- create a shard with master and replica1 only
- run InitShardMaster
- insert some data
- take a backup
- insert more data on the master
- bring up tablet_replica2 after the fact, let it restore the backup
- check all data is right (before+after backup data)
- list the backup, remove it
"""
for t in tablet_master, tablet_replica1:
t.create_db("vt_test_keyspace")
tablet_master.init_tablet("master", "test_keyspace", "0", start=True, supports_backups=True)
tablet_replica1.init_tablet("replica", "test_keyspace", "0", start=True, supports_backups=True)
utils.run_vtctl(["InitShardMaster", "test_keyspace/0", tablet_master.tablet_alias])
# insert data on master, wait for slave to get it
tablet_master.mquery("vt_test_keyspace", self._create_vt_insert_test)
self._insert_master(1)
timeout = 10
while True:
try:
result = tablet_replica1.mquery("vt_test_keyspace", "select count(*) from vt_insert_test")
if result[0][0] == 1:
break
except MySQLdb.DatabaseError:
# ignore exceptions, we'll just timeout (the tablet creation
# can take some time to replicate, and we get a 'table vt_insert_test
# does not exist exception in some rare cases)
logging.exception("exception waiting for data to replicate")
timeout = utils.wait_step("slave tablet getting data", timeout)
# backup the slave
utils.run_vtctl(["Backup", tablet_replica1.tablet_alias], auto_log=True)
# insert more data on the master
self._insert_master(2)
# now bring up the other slave, health check on, init_tablet on, restore on
tablet_replica2.start_vttablet(
wait_for_state="SERVING",
target_tablet_type="replica",
init_keyspace="test_keyspace",
init_shard="0",
supports_backups=True,
)
# check the new slave has the data
timeout = 10
while True:
result = tablet_replica2.mquery("vt_test_keyspace", "select count(*) from vt_insert_test")
if result[0][0] == 2:
break
timeout = utils.wait_step("new slave tablet getting data", timeout)
# list the backups
backups, _ = utils.run_vtctl(
tablet.get_backup_storage_flags() + ["ListBackups", "test_keyspace/0"],
mode=utils.VTCTL_VTCTL,
trap_output=True,
)
backups = backups.splitlines()
logging.debug("list of backups: %s", backups)
self.assertEqual(len(backups), 1)
self.assertTrue(backups[0].endswith(tablet_replica1.tablet_alias))
# remove the backup
utils.run_vtctl(
tablet.get_backup_storage_flags() + ["RemoveBackup", "test_keyspace/0", backups[0]],
auto_log=True,
mode=utils.VTCTL_VTCTL,
)
# make sure the list of backups is empty now
backups, _ = utils.run_vtctl(
tablet.get_backup_storage_flags() + ["ListBackups", "test_keyspace/0"],
mode=utils.VTCTL_VTCTL,
trap_output=True,
)
backups = backups.splitlines()
logging.debug("list of backups after remove: %s", backups)
self.assertEqual(len(backups), 0)
for t in tablet_master, tablet_replica1, tablet_replica2:
t.kill_vttablet()
