def mysql_backup(instance,
backup_type=backup.BACKUP_TYPE_XBSTREAM,
initial_build=False):
""" Run a file based backup on a supplied local instance
Args:
instance - A hostaddr object
backup_type - backup.BACKUP_TYPE_LOGICAL or backup.BACKUP_TYPE_XBSTREAM
initial_build - Boolean, if this is being created right after the server
was built
"""
log.info('Confirming sanity of replication (if applicable)')
zk = host_utils.MysqlZookeeper()
try:
(_, replica_type) = zk.get_replica_set_from_instance(instance)
except:
# instance is not in production
replica_type = None
if replica_type and replica_type != host_utils.REPLICA_ROLE_MASTER:
mysql_lib.assert_replication_sanity(instance)
log.info('Logging initial status to mysqlops')
start_timestamp = time.localtime()
lock_handle = None
backup_id = mysql_lib.start_backup_log(instance, backup_type,
start_timestamp)
# Take a lock to prevent multiple backups from running concurrently
try:
log.info('Taking backup lock')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
# Actually run the backup
log.info('Running backup')
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
backup_file = backup.xtrabackup_instance(instance, start_timestamp,
initial_build)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
backup_file = backup.logical_backup_instance(
instance, start_timestamp, initial_build)
else:
raise Exception('Unsupported backup type {backup_type}'
''.format(backup_type=backup_type))
finally:
if lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
# Update database with additional info now that backup is done.
if backup_id:
log.info("Updating database log entry with final backup info")
mysql_lib.finalize_backup_log(backup_id, backup_file)
else:
log.info("The backup is complete, but we were not able to "
"write to the central log DB.")
def mysql_backup(instance, backup_type=backup.BACKUP_TYPE_XBSTREAM):
""" Run a file based backup on a supplied local instance
Args:
instance - A hostaddr object
"""
log.info('Logging initial status to mysqlops')
start_timestamp = time.localtime()
lock_handle = None
backup_id = mysql_lib.start_backup_log(instance, backup_type,
start_timestamp)
# Take a lock to prevent multiple backups from running concurrently
try:
log.info('Taking backup lock')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
log.info('Cleaning up old backups')
purge_mysql_backups.purge_mysql_backups(instance, skip_lock=True)
# Actually run the backup
log.info('Running backup')
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
backup_file = backup.xtrabackup_instance(instance, start_timestamp)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
backup_file = backup.logical_backup_instance(
instance, start_timestamp)
else:
raise Exception('Unsupported backup type {backup_type}'
''.format(backup_type=backup_type))
# Upload file to s3
log.info('Uploading file to s3')
backup.s3_upload(backup_file)
finally:
if lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
# Update database with additional info now that backup is done.
if backup_id:
log.info("Updating database log entry with final backup info")
mysql_lib.finalize_backup_log(backup_id,
backup_file,
size=os.stat(backup_file).st_size)
else:
log.info("The backup is complete, but we were not able to "
"write to the central log DB.")
# Running purge again
log.info('Purging backups again')
purge_mysql_backups.purge_mysql_backups(instance)
def mysql_backup(instance, backup_type=backup.BACKUP_TYPE_XBSTREAM, initial_build=False):
""" Run a file based backup on a supplied local instance
Args:
instance - A hostaddr object
backup_type - backup.BACKUP_TYPE_LOGICAL or backup.BACKUP_TYPE_XBSTREAM
initial_build - Boolean, if this is being created right after the server
was built
"""
log.info('Confirming sanity of replication (if applicable)')
zk = host_utils.MysqlZookeeper()
try:
(_, replica_type) = zk.get_replica_set_from_instance(instance)
except:
# instance is not in production
replica_type = None
if replica_type and replica_type != host_utils.REPLICA_ROLE_MASTER:
mysql_lib.assert_replication_sanity(instance)
log.info('Logging initial status to mysqlops')
start_timestamp = time.localtime()
lock_handle = None
backup_id = mysql_lib.start_backup_log(instance, backup_type,
start_timestamp)
# Take a lock to prevent multiple backups from running concurrently
try:
log.info('Taking backup lock')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
# Actually run the backup
log.info('Running backup')
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
backup_file = backup.xtrabackup_instance(instance, start_timestamp, initial_build)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
backup_file = backup.logical_backup_instance(instance, start_timestamp, initial_build)
else:
raise Exception('Unsupported backup type {backup_type}'
''.format(backup_type=backup_type))
finally:
if lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
# Update database with additional info now that backup is done.
if backup_id:
log.info("Updating database log entry with final backup info")
mysql_lib.finalize_backup_log(backup_id, backup_file)
else:
log.info("The backup is complete, but we were not able to "
"write to the central log DB.")
def mysql_backup(instance, backup_type=backup.BACKUP_TYPE_XBSTREAM):
""" Run a file based backup on a supplied local instance
Args:
instance - A hostaddr object
"""
log.info("Logging initial status to mysqlops")
start_timestamp = time.localtime()
lock_handle = None
backup_id = mysql_lib.start_backup_log(instance, backup_type, start_timestamp)
# Take a lock to prevent multiple backups from running concurrently
try:
log.info("Taking backup lock")
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
log.info("Cleaning up old backups")
purge_mysql_backups.purge_mysql_backups(instance, skip_lock=True)
# Actually run the backup
log.info("Running backup")
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
backup_file = backup.xtrabackup_instance(instance, start_timestamp)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
backup_file = backup.logical_backup_instance(instance, start_timestamp)
else:
raise Exception("Unsupported backup type {backup_type}" "".format(backup_type=backup_type))
# Upload file to s3
log.info("Uploading file to s3")
backup.s3_upload(backup_file)
finally:
if lock_handle:
log.info("Releasing lock")
host_utils.release_flock_lock(lock_handle)
# Update database with additional info now that backup is done.
if backup_id:
log.info("Updating database log entry with final backup info")
mysql_lib.finalize_backup_log(backup_id, backup_file, size=os.stat(backup_file).st_size)
else:
log.info("The backup is complete, but we were not able to " "write to the central log DB.")
# Running purge again
log.info("Purging backups again")
purge_mysql_backups.purge_mysql_backups(instance)
def start_shard_migration(source_replica_set, destination_replica_set,
mig_dbs):
""" Move shards from one replica set to another
Args:
source_replica_set - Which replica set to take the shards from
destination_replica_set - Which replica set to put the shards on
mig_dbs - A set of databases to be migrated
"""
# In 2017Q1 shardb and modsharddb will learn how to deal with shard
# migrations. We will block them for now.
if source_replica_set.startswith('db') or \
source_replica_set.startswith('moddb'):
raise Exception('Sharddb and modsharddb migrations are not yet '
'supported')
if source_replica_set == destination_replica_set:
raise Exception('Source and destination can not be the same!')
# Dealing with failures, potentially due to failovers seems scary
# here. We are intentionally not catching exception as this seems racy
# and it would be far better for the entire process to fail than to mess
# with replication during a failover.
log.info('Requested to migrate from {s} to {d} databases: {db}'
''.format(s=source_replica_set,
d=destination_replica_set,
db=', '.join(mig_dbs)))
zk = host_utils.MysqlZookeeper()
source_master = zk.get_mysql_instance_from_replica_set(source_replica_set)
source_slave = zk.get_mysql_instance_from_replica_set(
source_replica_set, host_utils.REPLICA_ROLE_DR_SLAVE)
if not source_slave:
source_slave = zk.get_mysql_instance_from_replica_set(
source_replica_set, host_utils.REPLICA_ROLE_SLAVE)
log.info('Source host for dumping data {}'.format(source_slave))
destination_master = zk.get_mysql_instance_from_replica_set(
destination_replica_set)
log.info('Destination host for restoring data {}'
''.format(destination_master))
expected_dbs_on_source = zk.get_sharded_dbs_by_replica_set()[source_replica_set]
non_mig_dbs = mysql_lib.get_dbs(source_slave).difference(mig_dbs)
unexpected_dbs = mig_dbs.difference(expected_dbs_on_source)
if unexpected_dbs:
raise Exception('Unexpected database supplied for migraton: {}'
''.format(unexpected_dbs))
# Make sure there are no missing or extra shards
precheck_schema(source_master)
precheck_schema(destination_master)
# Check disk space
required_disk_space = get_required_disk_space(mig_dbs, source_master)
available_disk_space = disk_space_available_for_migration(destination_master)
if available_disk_space < required_disk_space:
raise Exception('Insufficent disk space to migrate, '
'available {a}MB, '
'requred {r}MB'
''.format(a=available_disk_space,
r=required_disk_space))
else:
log.info('Disk space looks ok: '
'available {a}MB, '
'requred {r}MB'
''.format(a=available_disk_space,
r=required_disk_space))
# Let's take out a lock to make sure we don't have multiple migrations
# running on the same replica sets (either source or destination).
lock_id = take_migration_lock(source_replica_set, destination_replica_set,
mig_dbs, non_mig_dbs)
try:
if(non_mig_dbs):
# First we will dump the schema for the shards that are not moving
log.info('Backing up non-migrating schema: {}'.format(non_mig_dbs))
no_mig_backup = backup.logical_backup_instance(
source_slave, time.localtime(),
blackhole=True, databases=non_mig_dbs)
time.sleep(1)
# And next the metadata db
log.info('Backing up metadata db: {}'.format(mysql_lib.METADATA_DB))
metadata_backup = backup.logical_backup_instance(
source_slave, time.localtime(),
databases=[mysql_lib.METADATA_DB])
time.sleep(1)
# Next we will backup the data for the shards that are moving
log.info('Backing up migrating schema data: {}'.format(mig_dbs))
mig_backup = backup.logical_backup_instance(
source_slave, time.localtime(),
databases=mig_dbs)
except:
finish_migration_log(lock_id, STATUS_EXPORT_FAILED)
raise
if(non_mig_dbs):
# Finally import the backups
log.info('Importing all the blackhole tables')
mysql_restore.logical_restore(no_mig_backup, destination_master)
log.info('Import metadata')
mysql_restore.logical_restore(metadata_backup, destination_master)
log.info('Setting up replication')
mysql_lib.change_master(destination_master, source_master,
'BOGUS', 0, no_start=True, skip_set_readonly=True,
gtid_auto_pos=False)
mysql_restore.logical_restore(mig_backup, destination_master)
# add start slave, catchup
mysql_lib.start_replication(destination_master)
mysql_lib.wait_for_catch_up(destination_master, migration=True)
# And update the log/locks
update_migration_status(lock_id, STATUS_FAILOVER_READY)
log.info('The migration is ready to be finished by running:')
log.info('/usr/local/bin/mysql_utils/finish_shard_migration.py {src}'
''.format(src=source_replica_set))
def mysql_backup(instance,
backup_type=backup.BACKUP_TYPE_XBSTREAM,
initial_build=False,
lock_handle=None):
""" Run a file based backup on a supplied local instance
Args:
instance - A hostaddr object
backup_type - backup.BACKUP_TYPE_LOGICAL or backup.BACKUP_TYPE_XBSTREAM
initial_build - Boolean, if this is being created right after the server
was built
lock_handle - A lock handle, if we have one from the caller.
"""
if backup_type == backup.BACKUP_TYPE_XBSTREAM and \
os.path.isfile(backup.XTRABACKUP_SKIP_FILE):
log.info('Found {}. Skipping xtrabackup '
'run.'.format(backup.XTRABACKUP_SKIP_FILE))
return
log.info('Confirming sanity of replication (if applicable)')
zk = host_utils.MysqlZookeeper()
try:
replica_type = zk.get_replica_type_from_instance(instance)
except:
# instance is not in production
replica_type = None
if replica_type and replica_type != host_utils.REPLICA_ROLE_MASTER:
mysql_lib.assert_replication_sanity(instance)
log.info('Logging initial status to mysqlops')
start_timestamp = time.localtime()
backup_id = mysql_lib.start_backup_log(instance, backup_type,
start_timestamp)
# Take a lock to prevent multiple backups from running concurrently
# unless we already have a lock from the caller. This means we
# also don't have to release the lock at the end; either we
# exit the script entirely, and it gets cleaned up or the caller
# maintains it.
if lock_handle is None:
log.info('Taking backup lock')
lock_handle = host_utils.bind_lock_socket(
backup.STD_BACKUP_LOCK_SOCKET)
else:
log.info('Not acquiring backup lock, we already have one.')
# Actually run the backup
log.info('Running backup')
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
backup_file = backup.xtrabackup_instance(instance, start_timestamp,
initial_build)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
# We don't need a backup-skip file here since this isn't
# regularly scheduled.
backup_file = backup.logical_backup_instance(instance, start_timestamp,
initial_build)
else:
raise Exception('Unsupported backup type {}'.format(backup_type))
# Update database with additional info now that backup is done.
if backup_id:
log.info("Updating database log entry with final backup info")
mysql_lib.finalize_backup_log(backup_id, backup_file)
else:
log.info("The backup is complete, but we were not able to "
"write to the central log DB.")
