def logical_restore(dump, destination):
""" Restore a compressed mysqldump file from s3 to localhost, port 3306
Args:
dump - a mysqldump file in s3
destination - a hostaddr object for where the data should be loaded on
localhost
"""
(user, password) = mysql_lib.get_mysql_user_for_role('admin')
if dump.name.startswith(backup.BACKUP_TYPE_PARTIAL_LOGICAL):
# TODO: check if db is empty before applying rate limit
rate_limit = backup.MAX_TRANSFER_RATE
else:
log.info('Restarting MySQL to turn off enforce_storage_engine')
host_utils.stop_mysql(destination.port)
host_utils.start_mysql(destination.port,
host_utils.DEFAULTS_FILE_ARG.format(
defaults_file=host_utils.MYSQL_UPGRADE_CNF_FILE))
rate_limit = None
log.info('Downloading, decompressing and importing backup')
procs = dict()
procs['s3_download'] = backup.create_s3_download_proc(dump)
procs['pv'] = backup.create_pv_proc(procs['s3_download'].stdout,
size=dump.size,
rate_limit=rate_limit)
log.info('zcat |')
procs['zcat'] = subprocess.Popen(['zcat'],
stdin=procs['pv'].stdout,
stdout=subprocess.PIPE)
mysql_cmd = ['mysql',
'--port={}'.format(str(destination.port)),
'--host={}'.format(destination.hostname),
'--user={}'.format(user),
'--password={}'.format(password)]
log.info(' '.join(mysql_cmd))
procs['mysql'] = subprocess.Popen(mysql_cmd,
stdin=procs['zcat'].stdout)
while(not host_utils.check_dict_of_procs(procs)):
time.sleep(.5)
def logical_restore(dump, destination):
""" Restore a compressed mysqldump file from s3 to localhost, port 3306
Args:
dump - a mysqldump file in s3
destination - a hostaddr object for where the data should be loaded on
localhost
"""
log.info('Preparing replication')
(restore_source, _) = backup.get_metadata_from_backup_file(dump.name)
# We are importing a mysqldump which was created with --master-data
# so there will be a CHANGE MASTER statement at the start of the dump.
# MySQL will basically just ignore a CHANGE MASTER command if
# master_host is not already setup. So we are setting master_host,
# username and password here. We use BOGUS for master_log_file so that
# the IO thread is intentionally broken. With no argument for
# master_log_file, the IO thread would start downloading the first bin log
# and the SQL thread would start executing...
mysql_lib.change_master(destination, restore_source, 'BOGUS', 0,
no_start=True)
log.info('Restarting MySQL to turn off enforce_storage_engine')
host_utils.stop_mysql(destination.port)
host_utils.start_mysql(destination.port,
host_utils.DEFAULTS_FILE_ARG.format(defaults_file=host_utils.MYSQL_UPGRADE_CNF_FILE))
log.info('Downloading, decompressing and importing backup')
procs = dict()
procs['s3_download'] = backup.create_s3_download_proc(dump)
procs['pv'] = backup.create_pv_proc(procs['s3_download'].stdout,
size=dump.size)
log.info('zcat |')
procs['zcat'] = subprocess.Popen(['zcat'],
stdin=procs['pv'].stdout,
stdout=subprocess.PIPE)
mysql_cmd = ['mysql', '--port', str(destination.port)]
log.info(' '.join(mysql_cmd))
procs['mysql'] = subprocess.Popen(mysql_cmd,
stdin=procs['zcat'].stdout,
stdout=subprocess.PIPE)
while(not host_utils.check_dict_of_procs(procs)):
time.sleep(.5)
def restore_instance(backup_type, restore_source, destination,
no_repl, date,
add_to_zk, skip_production_check):
""" Restore a MySQL backup on to localhost
Args:
backup_type - Type of backup to restore
restore_source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
no_repl - Should replication be not started. It will always be setup.
date - What date should the backup be from
add_to_zk - Should the instnace be added to zk. If so, the log from the
host being launched will be consulted.
skip_production_check - Do not check if the host is already in zk for
production use.
"""
log.info('Supplied source is {source}'.format(source=restore_source))
log.info('Supplied destination is {dest}'.format(dest=destination))
log.info('Desired date of restore {date}'.format(date=date))
zk = host_utils.MysqlZookeeper()
# Try to prevent unintentional destruction of prod servers
log.info('Confirming no prod instances running on destination')
prod_check(destination, skip_production_check)
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a lock to block another restore from starting')
lock_handle = host_utils.bind_lock_socket(backup.STD_BACKUP_LOCK_SOCKET)
log.info('Looking for a backup to restore')
if restore_source:
possible_sources = [restore_source]
else:
possible_sources = get_possible_sources(destination, backup_type)
backup_key = find_a_backup_to_restore(possible_sources, destination,
backup_type, date)
# Figure out what what we use to as the master when we setup replication
(restore_source, _) = backup.get_metadata_from_backup_file(backup_key.name)
try:
replica_set = restore_source.get_zk_replica_set()
master = zk.get_mysql_instance_from_replica_set(replica_set,
host_utils.REPLICA_ROLE_MASTER)
except:
# ZK has no idea what this replica set is, probably a new replica set.
master = restore_source
# Start logging
row_id = backup.start_restore_log(master, {
'restore_source': restore_source,
'restore_port': destination.port,
'restore_file': backup_key.name,
'source_instance': destination.hostname,
'restore_date': date,
'replication': no_repl,
'zookeeper': add_to_zk})
# Giant try to allow logging if anything goes wrong.
try:
# If we hit an exception, this status will be used. If not, it will
# be overwritten
restore_log_update = {'restore_status': 'BAD'}
# This also ensures that all needed directories exist
log.info('Rebuilding local mysql instance')
lock_handle = mysql_init_server.mysql_init_server(
destination,
skip_production_check=True,
skip_backup=True,
lock_handle=lock_handle)
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
xbstream_restore(backup_key, destination.port)
if master == restore_source:
log.info('Pulling replication info for restore from '
'backup source')
(binlog_file,
binlog_pos,
gtid_purged) = backup.parse_xtrabackup_binlog_info(
destination.port)
else:
log.info('Pulling replication info for restore from '
'master of backup source')
# if our backup came from a GTID server, we won't have
# a binlog_file and a binlog_pos, so we need to see if
# we can get a set of purged GTIDs
(binlog_file,
binlog_pos,
gtid_purged) = backup.parse_xtrabackup_slave_info(
destination.port)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
log.info('Preparing replication')
# We are importing a mysqldump which was created with
# --master-data or --dump-slave so there will be a CHANGE MASTER
# statement at the start of the dump. MySQL will basically just
# ignore a CHANGE MASTER command if master_host is not already
# setup. So we are setting master_host, username and password
# here. We use BOGUS for master_log_file so that the IO thread is
# intentionally broken. With no argument for master_log_file,
# the IO thread would start downloading the first bin log and
# the SQL thread would start executing...
mysql_lib.change_master(destination, master, 'BOGUS', 0,
no_start=True)
# reset master on slave before we load anything to ensure that
# we can set GTID info from the backup, if it exists.
mysql_lib.reset_master(destination)
logical_restore(backup_key, destination)
host_utils.stop_mysql(destination.port)
log.info('Running MySQL upgrade')
host_utils.upgrade_auth_tables(destination.port)
log.info('Starting MySQL')
host_utils.start_mysql(
destination.port,
options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(
defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))
# Since we haven't started the slave yet, make sure we've got these
# plugins installed, whether we use them or not.
mysql_lib.setup_semisync_plugins(destination)
mysql_lib.setup_audit_plugin(destination)
mysql_lib.setup_response_time_metrics(destination)
restore_log_update = {'restore_status': 'OK'}
# Try to configure replication.
log.info('Setting up MySQL replication')
restore_log_update['replication'] = 'FAIL'
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
# before we change master, reset master on the
# slave to clear out any GTID errant transactions.
mysql_lib.reset_master(destination)
mysql_lib.change_master(destination,
master,
binlog_file,
binlog_pos,
gtid_purged=gtid_purged,
no_start=(no_repl == 'SKIP'))
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
if no_repl == 'SKIP':
log.info('As requested, not starting replication.')
else:
mysql_lib.restart_replication(destination)
if no_repl == 'REQ':
mysql_lib.wait_for_catch_up(destination)
restore_log_update['replication'] = 'OK'
host_utils.manage_pt_daemons(destination.port)
except Exception as e:
log.error(e)
if row_id is not None:
restore_log_update['status_message'] = e
restore_log_update['finished_at'] = True
raise
finally:
# As with mysql_init_server, we have to do one more restart to
# clear out lock ownership, but here we have to also do it with
# the proper config file.
if lock_handle:
log.info('Releasing lock and restarting MySQL')
host_utils.stop_mysql(destination.port)
time.sleep(5)
host_utils.release_lock_socket(lock_handle)
if no_repl == 'SKIP':
host_utils.start_mysql(
destination.port,
options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(
defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))
else:
host_utils.start_mysql(destination.port)
backup.update_restore_log(master, row_id, restore_log_update)
try:
if add_to_zk == 'REQ':
if no_repl == 'REQ':
log.info('Waiting for replication again, as it may have '
'drifted due to restart.')
mysql_lib.wait_for_catch_up(destination)
log.info('Waiting for IO lag in case it is still too '
'far even wait for resync ')
mysql_lib.wait_for_catch_up(destination, io=True)
log.info('Adding instance to zk.')
modify_mysql_zk.auto_add_instance_to_zk(destination.port,
dry_run=False)
backup.update_restore_log(master, row_id, {'zookeeper': 'OK'})
else:
log.info('add_to_zk is not set, therefore not adding to zk')
except Exception as e:
log.warning("An exception occurred: {}".format(e))
log.warning("If this is a DB issue, that's fine. "
"Otherwise, you should check ZK.")
backup.update_restore_log(master, row_id, {'finished_at': True})
if no_repl == 'REQ':
log.info('Starting a new backup')
mysql_backup.mysql_backup(destination, initial_build=True)
def restore_instance(restore_source, destination, restore_type,
restore_file, no_repl, date,
add_to_zk, skip_production_check,
test_restore):
""" Restore a MySQL backup on to localhost
Args:
restore_source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
restore_type - How to pull the backup, options are 's3', 'remote_server'
and 'local_file'
no_repl - Should replication be not started. It will always be setup.
date - What date should the backup be from
add_to_zk - Should the instnace be added to zk. If so, the log from the
host being launched will be consulted.
skip_production_check - Do not check if the host is already in zk for
production use.
test_restore - Use less ram and shutdown the instance after going
through the motions of a restore.
"""
(temp_dir, target_dir) = backup.get_paths(str(destination.port))
log.info('Supplied source is {source}'.format(source=restore_source))
log.info('Supplied destination is {dest}'.format(dest=destination))
log.info('Restore type is {rest}'.format(rest=restore_type))
log.info('Local restore file is {file}'.format(file=restore_file))
log.info('Desired date of restore {date}'.format(date=date))
if test_restore == 'test':
log.info('Running restore in test mode')
# Try to prevent unintentional destruction of prod servers
zk = host_utils.MysqlZookeeper()
try:
(_, replica_type) = zk.get_replica_set_from_instance(destination)
except:
# instance is not in production
replica_type = None
if replica_type == host_utils.REPLICA_ROLE_MASTER:
# If the instance, we will refuse to run. No ifs, ands, or buts/
raise Exception('Restore script must never run on a master')
if replica_type:
if skip_production_check:
log.info('Ignoring production check. We hope you know what you '
'are doing and we will try to take a backup in case '
'you are wrong.')
try:
mysql_backup.mysql_backup(destination)
except Exception as e:
log.error(e)
log.warning('Unable to take a backup. We will give you {time} '
'seconds to change your mind and ^c.'
''.format(time=SCARY_TIMEOUT))
time.sleep(SCARY_TIMEOUT)
else:
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=destination))
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block another restore from starting')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
log.info('Rebuilding cnf files just in case')
mysql_cnf_builder.build_cnf()
mysql_init_server.create_and_chown_dirs(destination.port)
# load some data from the mysql conf file
datadir = host_utils.get_cnf_setting('datadir', destination.port)
# Where will we look for a backup?
if restore_type != 'local_file':
(restore_type, restore_source,
restore_file, restore_size) = find_a_backup_to_restore(restore_type, restore_source,
destination, date)
# Not using an if/else because find_a_backup_to_restore could set to
# local_file if the file has already been downloaded.
if restore_type == 'local_file':
restore_source = backup.get_host_from_backup(restore_file)
# restore_size will be computed in the unpack function
restore_size = None
log.info('Detected the source of backup as {src}'.format(src=restore_source))
if restore_source.get_zk_replica_set():
replica_set = restore_source.get_zk_replica_set()[0]
master = zk.get_mysql_instance_from_replica_set(replica_set, host_utils.REPLICA_ROLE_MASTER)
else:
# ZK has no idea what this replica set is, probably a new replica set.
master = restore_source
# Start logging
row_id = backup.start_restore_log(master, {'restore_type': restore_type,
'test_restore': test_restore,
'restore_source': restore_source,
'restore_port': destination.port,
'restore_file': restore_file,
'source_instance': destination.hostname,
'restore_date': date,
'replication': no_repl,
'zookeeper': add_to_zk})
# Giant try to allow logging if anything goes wrong.
try:
# If we hit an exception, this status will be used. If not, it will
# be overwritten
restore_log_update = {'restore_status': 'BAD'}
log.info('Quick sanity check')
mysql_init_server.basic_host_sanity()
log.info('Shutting down MySQL')
host_utils.stop_mysql(destination.port)
log.info('Removing any existing MySQL data')
mysql_init_server.delete_mysql_data(destination.port)
log.info('Unpacking {rfile} into {ddir}'.format(rfile=restore_file,
ddir=datadir))
backup.xbstream_unpack(restore_file, destination.port,
restore_source, restore_type, restore_size)
log.info('Decompressing files in {path}'.format(path=datadir))
backup.innobackup_decompress(destination.port)
log.info('Applying logs')
if test_restore == 'test':
# We don't really need a lot of memory if we're just
# verifying that it works.
backup.apply_log(destination.port, memory='1G')
else:
backup.apply_log(destination.port, memory='10G')
log.info('Removing old innodb redo logs')
mysql_init_server.delete_innodb_log_files(destination.port)
log.info('Setting permissions for MySQL on {dir}'.format(dir=datadir))
host_utils.change_owner(datadir, 'mysql', 'mysql')
log.info('Starting MySQL')
host_utils.upgrade_auth_tables(destination.port)
restore_log_update = {'restore_status': 'OK'}
log.info('Running MySQL upgrade')
host_utils.start_mysql(destination.port,
options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))
if master == backup.get_host_from_backup(restore_file):
log.info('Pulling replication info from restore to backup source')
(binlog_file, binlog_pos) = backup.parse_xtrabackup_binlog_info(datadir)
else:
log.info('Pulling replication info from restore to '
'master of backup source')
(binlog_file, binlog_pos) = backup.parse_xtrabackup_slave_info(datadir)
log.info('Setting up MySQL replication')
restore_log_update['replication'] = 'FAIL'
# Since we haven't started the slave yet, make sure we've got these
# plugins installed, whether we use them or not.
mysql_lib.setup_semisync_plugins(destination)
# Try to configure replication. If this was just a test restore,
# don't wait for it to catch up - don't even start the slave.
if test_restore == 'test':
mysql_lib.change_master(destination,
master,
binlog_file,
binlog_pos,
no_start=True)
backup.quick_test_replication(destination)
else:
mysql_lib.change_master(destination,
master,
binlog_file,
binlog_pos,
no_start=(no_repl == 'SKIP'))
mysql_lib.wait_replication_catch_up(destination)
host_utils.restart_pt_daemons(destination.port)
restore_log_update['replication'] = 'OK'
mysql_lib.setup_response_time_metrics(destination)
except Exception as e:
log.error(e)
if row_id is not None:
restore_log_update['status_message'] = e
restore_log_update['finished_at'] = True
raise
finally:
if lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
backup.update_restore_log(master, row_id, restore_log_update)
# If this was a test restore, we don't need to keep the 3307
# instance running, so let's shut it off.
if test_restore == 'test':
log.info('Shutting down MySQL backup/restore test instance')
host_utils.stop_mysql(destination.port)
backup.update_restore_log(master, row_id, {'finished_at': True})
return
try:
if add_to_zk == 'REQ':
log.info('Adding instance to zk')
modify_mysql_zk.auto_add_instance_to_zk(destination, dry_run=False)
backup.update_restore_log(master, row_id, {'zookeeper': 'OK'})
else:
log.info('add_to_zk is not set, therefore not adding to zk')
except Exception as e:
log.warning("An exception occurred: {e}".format(e=e))
log.warning("If this is a DB issue, that's fine. "
"Otherwise, you should check ZK.")
backup.update_restore_log(master, row_id, {'finished_at': True})
log.info('Starting a new backup')
mysql_backup.mysql_backup(destination)
def mysql_init_server(instance,
skip_production_check=False,
skip_backup=True,
lock_handle=None):
""" Remove any data and initialize a MySQL instance
Args:
instance - A hostaddr object pointing towards localhost to act upon
skip_production_check - Dangerous! will not run safety checks to protect
production data
skip_backup - Don't run a backup after the instance is setup
lock_handle - If the caller already locked the system, pass in the
lock handle, as we may need to release and reacquire
to prevent mysqld from keeping it.
"""
if lock_handle is None:
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a lock to block race conditions')
lock_handle = host_utils.bind_lock_socket(
backup.STD_BACKUP_LOCK_SOCKET)
else:
log.info('Lock already exists from caller.')
try:
# sanity check
zk = host_utils.MysqlZookeeper()
if (not skip_production_check
and instance in zk.get_all_mysql_instances()):
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=instance))
log.info('Checking host for mounts, etc...')
basic_host_sanity()
log.info('(re)Generating MySQL cnf files')
mysql_cnf_builder.build_cnf()
log.info('Creating any missing directories')
create_and_chown_dirs(instance.port)
log.info('Shutting down MySQL (if applicable)')
host_utils.stop_mysql(instance.port)
log.info('Deleting existing MySQL data')
delete_mysql_data(instance.port)
log.info('Creating MySQL privileges tables')
init_privileges_tables(instance.port)
log.info('Clearing innodb log files')
delete_innodb_log_files(instance.port)
log.info('Starting up instance')
host_utils.start_mysql(instance.port)
log.info('Importing MySQL users')
mysql_grants.manage_mysql_grants(instance, 'nuke_then_import')
log.info('Creating test database')
mysql_lib.create_db(instance, 'test')
log.info('Setting up query response time plugins')
mysql_lib.setup_response_time_metrics(instance)
log.info('Setting up semi-sync replication plugins')
mysql_lib.setup_semisync_plugins(instance)
log.info('Setting up audit log plugin')
mysql_lib.setup_audit_plugin(instance)
log.info('Restarting pt daemons')
host_utils.manage_pt_daemons(instance.port)
log.info('MySQL initalization complete')
finally:
# We have to do this, ugly though it may be, to ensure that
# the running MySQL process doesn't maintain a hold on the lock
# socket after the script exits. We reacquire the lock after
# the restart and pass it back to the caller.
#
if lock_handle:
log.info('Restarting MySQL, releasing lock.')
host_utils.stop_mysql(instance.port)
log.info('Sleeping 5 seconds.')
time.sleep(5)
host_utils.release_lock_socket(lock_handle)
host_utils.start_mysql(instance.port)
log.info('Reacquiring lock.')
lock_handle = host_utils.bind_lock_socket(
backup.STD_BACKUP_LOCK_SOCKET)
if not skip_backup:
log.info('Taking a backup')
mysql_backup.mysql_backup(instance,
initial_build=True,
lock_handle=lock_handle)
return lock_handle
def restore_instance(backup_type, restore_source, destination,
no_repl, date,
add_to_zk, skip_production_check):
""" Restore a MySQL backup on to localhost
Args:
backup_type - Type of backup to restore
restore_source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
no_repl - Should replication be not started. It will always be setup.
date - What date should the backup be from
add_to_zk - Should the instnace be added to zk. If so, the log from the
host being launched will be consulted.
skip_production_check - Do not check if the host is already in zk for
production use.
"""
log.info('Supplied source is {source}'.format(source=restore_source))
log.info('Supplied destination is {dest}'.format(dest=destination))
log.info('Desired date of restore {date}'.format(date=date))
zk = host_utils.MysqlZookeeper()
# Try to prevent unintentional destruction of prod servers
log.info('Confirming no prod instances running on destination')
prod_check(destination, skip_production_check)
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block another restore from starting')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
log.info('Looking for a backup to restore')
if restore_source:
possible_sources = [restore_source]
else:
possible_sources = get_possible_sources(destination, backup_type)
backup_key = find_a_backup_to_restore(possible_sources, destination,
backup_type, date)
# Figure out what what we use to as the master when we setup replication
(restore_source, _) = backup.get_metadata_from_backup_file(backup_key.name)
if restore_source.get_zk_replica_set():
replica_set = restore_source.get_zk_replica_set()[0]
master = zk.get_mysql_instance_from_replica_set(replica_set, host_utils.REPLICA_ROLE_MASTER)
else:
# ZK has no idea what this replica set is, probably a new replica set.
master = restore_source
# Start logging
row_id = backup.start_restore_log(master, {'restore_source': restore_source,
'restore_port': destination.port,
'restore_file': backup_key.name,
'source_instance': destination.hostname,
'restore_date': date,
'replication': no_repl,
'zookeeper': add_to_zk})
# Giant try to allow logging if anything goes wrong.
try:
# If we hit an exception, this status will be used. If not, it will
# be overwritten
restore_log_update = {'restore_status': 'BAD'}
# This also ensures that all needed directories exist
log.info('Rebuilding local mysql instance')
mysql_init_server.mysql_init_server(destination, skip_production_check=True,
skip_backup=True, skip_locking=True)
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
xbstream_restore(backup_key, destination.port)
if master == restore_source:
log.info('Pulling replication info from restore to backup source')
(binlog_file, binlog_pos) = backup.parse_xtrabackup_binlog_info(destination.port)
else:
log.info('Pulling replication info from restore to '
'master of backup source')
(binlog_file, binlog_pos) = backup.parse_xtrabackup_slave_info(destination.port)
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
logical_restore(backup_key, destination)
host_utils.stop_mysql(destination.port)
log.info('Running MySQL upgrade')
host_utils.upgrade_auth_tables(destination.port)
log.info('Starting MySQL')
host_utils.start_mysql(destination.port,
options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))
# Since we haven't started the slave yet, make sure we've got these
# plugins installed, whether we use them or not.
mysql_lib.setup_semisync_plugins(destination)
restore_log_update = {'restore_status': 'OK'}
# Try to configure replication.
log.info('Setting up MySQL replication')
restore_log_update['replication'] = 'FAIL'
if backup_type == backup.BACKUP_TYPE_XBSTREAM:
mysql_lib.change_master(destination,
master,
binlog_file,
binlog_pos,
no_start=(no_repl == 'SKIP'))
elif backup_type == backup.BACKUP_TYPE_LOGICAL:
if no_repl == 'SKIP':
log.info('As requested, not starting replication.')
else:
mysql_lib.restart_replication(destination)
if no_repl == 'REQ':
mysql_lib.wait_replication_catch_up(destination)
restore_log_update['replication'] = 'OK'
host_utils.restart_pt_daemons(destination.port)
mysql_lib.setup_response_time_metrics(destination)
except Exception as e:
log.error(e)
if row_id is not None:
restore_log_update['status_message'] = e
restore_log_update['finished_at'] = True
raise
finally:
if lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
backup.update_restore_log(master, row_id, restore_log_update)
try:
if add_to_zk == 'REQ':
log.info('Adding instance to zk')
modify_mysql_zk.auto_add_instance_to_zk(destination.port,
dry_run=False)
backup.update_restore_log(master, row_id, {'zookeeper': 'OK'})
else:
log.info('add_to_zk is not set, therefore not adding to zk')
except Exception as e:
log.warning("An exception occurred: {e}".format(e=e))
log.warning("If this is a DB issue, that's fine. "
"Otherwise, you should check ZK.")
backup.update_restore_log(master, row_id, {'finished_at': True})
if no_repl == 'REQ':
log.info('Starting a new backup')
mysql_backup.mysql_backup(destination, initial_build=True)
def mysql_init_server(instance,
skip_production_check=False,
skip_locking=False,
skip_backup=True):
""" Remove any data and initialize a MySQL instance
Args:
instance - A hostaddr object pointing towards localhost to act upon
skip_production_check - Dangerous! will not run safety checks to protect
production data
skip_locking - Do not take a lock on localhost. Useful when the caller has
already has taken the lock (ie mysql_restore_xtrabackup)
skip_backup - Don't run a backup after the instance is setup
"""
lock_handle = None
if not skip_locking:
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block race conditions')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
try:
# sanity check
zk = host_utils.MysqlZookeeper()
if (not skip_production_check
and instance in zk.get_all_mysql_instances()):
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=instance))
log.info('Checking host for mounts, etc...')
basic_host_sanity()
log.info('(re)Generating MySQL cnf files')
mysql_cnf_builder.build_cnf()
log.info('Creating any missing directories')
create_and_chown_dirs(instance.port)
log.info('Shutting down MySQL (if applicable)')
host_utils.stop_mysql(instance.port)
log.info('Deleting existing MySQL data')
delete_mysql_data(instance.port)
log.info('Creating MySQL privileges tables')
init_privileges_tables(instance.port)
log.info('Clearing innodb log files')
delete_innodb_log_files(instance.port)
log.info('Starting up instance')
host_utils.start_mysql(instance.port)
log.info('Importing MySQL users')
mysql_grants.manage_mysql_grants(instance, 'nuke_then_import')
log.info('Creating test database')
mysql_lib.create_db(instance, 'test')
log.info('Setting up query response time plugins')
mysql_lib.setup_response_time_metrics(instance)
log.info('Setting up semi-sync replication plugins')
mysql_lib.setup_semisync_plugins(instance)
log.info('Restarting pt daemons')
host_utils.restart_pt_daemons(instance.port)
log.info('MySQL initalization complete')
finally:
if not skip_locking and lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
if not skip_backup:
log.info('Taking a backup')
mysql_backup.mysql_backup(instance)
def mysql_init_server(instance,
skip_production_check=False, skip_locking=False,
skip_backup=True):
""" Remove any data and initialize a MySQL instance
Args:
instance - A hostaddr object pointing towards localhost to act upon
skip_production_check - Dangerous! will not run safety checks to protect
production data
skip_locking - Do not take a lock on localhost. Useful when the caller has
already has taken the lock (ie mysql_restore_xtrabackup)
skip_backup - Don't run a backup after the instance is setup
"""
lock_handle = None
if not skip_locking:
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block race conditions')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
try:
# sanity check
zk = host_utils.MysqlZookeeper()
if (not skip_production_check and
instance in zk.get_all_mysql_instances()):
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=instance))
log.info('Checking host for mounts, etc...')
basic_host_sanity()
log.info('(re)Generating MySQL cnf files')
mysql_cnf_builder.build_cnf()
log.info('Creating any missing directories')
create_and_chown_dirs(instance.port)
log.info('Shutting down MySQL (if applicable)')
host_utils.stop_mysql(instance.port)
log.info('Deleting existing MySQL data')
delete_mysql_data(instance.port)
log.info('Creating MySQL privileges tables')
init_privileges_tables(instance.port)
log.info('Clearing innodb log files')
delete_innodb_log_files(instance.port)
log.info('Starting up instance')
host_utils.start_mysql(instance.port)
log.info('Importing MySQL users')
mysql_grants.manage_mysql_grants(instance, 'nuke_then_import')
log.info('Creating test database')
conn = mysql_lib.connect_mysql(instance)
mysql_lib.create_db(conn, 'test')
log.info('Setting up query response time plugins')
mysql_lib.setup_response_time_metrics(instance)
log.info('Setting up semi-sync replication plugins')
mysql_lib.setup_semisync_plugins(instance)
log.info('Restarting pt daemons')
host_utils.restart_pt_daemons(instance.port)
log.info('MySQL initalization complete')
finally:
if not skip_locking and lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
if not skip_backup:
log.info('Taking a backup')
mysql_backup.mysql_backup(instance)
def restore_instance(restore_source, destination, no_repl, date, add_to_zk,
skip_production_check):
""" Restore a MySQL backup on to localhost
Args:
restore_source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
no_repl - Should replication be not started. It will always be setup.
date - What date should the backup be from
add_to_zk - Should the instnace be added to zk. If so, the log from the
host being launched will be consulted.
skip_production_check - Do not check if the host is already in zk for
production use.
"""
log.info('Supplied source is {source}'.format(source=restore_source))
log.info('Supplied destination is {dest}'.format(dest=destination))
log.info('Desired date of restore {date}'.format(date=date))
# Try to prevent unintentional destruction of prod servers
zk = host_utils.MysqlZookeeper()
try:
(_, replica_type) = zk.get_replica_set_from_instance(destination)
except:
# instance is not in production
replica_type = None
if replica_type == host_utils.REPLICA_ROLE_MASTER:
# If the instance, we will refuse to run. No ifs, ands, or buts/
raise Exception('Restore script must never run on a master')
if replica_type:
if skip_production_check:
log.info('Ignoring production check. We hope you know what you '
'are doing and we will try to take a backup in case '
'you are wrong.')
try:
mysql_backup.mysql_backup(destination)
except Exception as e:
log.error(e)
log.warning('Unable to take a backup. We will give you {time} '
'seconds to change your mind and ^c.'
''.format(time=SCARY_TIMEOUT))
time.sleep(SCARY_TIMEOUT)
else:
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=destination))
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block another restore from starting')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
log.info('Rebuilding cnf files just in case')
mysql_cnf_builder.build_cnf()
mysql_init_server.create_and_chown_dirs(destination.port)
# load some data from the mysql conf file
datadir = host_utils.get_cnf_setting('datadir', destination.port)
(restore_source, restore_file,
restore_size) = find_a_backup_to_restore(restore_source, destination,
date)
if restore_source.get_zk_replica_set():
replica_set = restore_source.get_zk_replica_set()[0]
master = zk.get_mysql_instance_from_replica_set(
replica_set, host_utils.REPLICA_ROLE_MASTER)
else:
# ZK has no idea what this replica set is, probably a new replica set.
master = restore_source
# Start logging
row_id = backup.start_restore_log(
master, {
'restore_source': restore_source,
'restore_port': destination.port,
'restore_file': restore_file,
'source_instance': destination.hostname,
'restore_date': date,
'replication': no_repl,
'zookeeper': add_to_zk
})
# Giant try to allow logging if anything goes wrong.
try:
# If we hit an exception, this status will be used. If not, it will
# be overwritten
restore_log_update = {'restore_status': 'BAD'}
log.info('Quick sanity check')
mysql_init_server.basic_host_sanity()
log.info('Shutting down MySQL')
host_utils.stop_mysql(destination.port)
log.info('Removing any existing MySQL data')
mysql_init_server.delete_mysql_data(destination.port)
log.info('Unpacking {rfile} into {ddir}'.format(rfile=restore_file,
ddir=datadir))
backup.xbstream_unpack(restore_file, destination.port, restore_source,
restore_size)
log.info('Decompressing files in {path}'.format(path=datadir))
backup.innobackup_decompress(destination.port)
# Determine how much RAM to use for applying logs based on the
# system's total RAM size; all our boxes have 32G or more, so
# this will always be better than before, but not absurdly high.
log_apply_ram = psutil.phymem_usage()[0] / 1024 / 1024 / 1024 / 3
log.info('Applying logs')
backup.apply_log(destination.port, memory='{}G'.format(log_apply_ram))
log.info('Removing old innodb redo logs')
mysql_init_server.delete_innodb_log_files(destination.port)
log.info('Setting permissions for MySQL on {dir}'.format(dir=datadir))
host_utils.change_owner(datadir, 'mysql', 'mysql')
log.info('Starting MySQL')
host_utils.upgrade_auth_tables(destination.port)
restore_log_update = {'restore_status': 'OK'}
log.info('Running MySQL upgrade')
host_utils.start_mysql(
destination.port,
options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(
defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))
if master == backup.get_metadata_from_backup_file(restore_file)[0]:
log.info('Pulling replication info from restore to backup source')
(binlog_file,
binlog_pos) = backup.parse_xtrabackup_binlog_info(datadir)
else:
log.info('Pulling replication info from restore to '
'master of backup source')
(binlog_file,
binlog_pos) = backup.parse_xtrabackup_slave_info(datadir)
log.info('Setting up MySQL replication')
restore_log_update['replication'] = 'FAIL'
# Since we haven't started the slave yet, make sure we've got these
# plugins installed, whether we use them or not.
mysql_lib.setup_semisync_plugins(destination)
# Try to configure replication.
mysql_lib.change_master(destination,
master,
binlog_file,
binlog_pos,
no_start=(no_repl == 'SKIP'))
mysql_lib.wait_replication_catch_up(destination)
host_utils.restart_pt_daemons(destination.port)
restore_log_update['replication'] = 'OK'
mysql_lib.setup_response_time_metrics(destination)
except Exception as e:
log.error(e)
if row_id is not None:
restore_log_update['status_message'] = e
restore_log_update['finished_at'] = True
raise
finally:
if lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
backup.update_restore_log(master, row_id, restore_log_update)
try:
if add_to_zk == 'REQ':
log.info('Adding instance to zk')
modify_mysql_zk.auto_add_instance_to_zk(destination, dry_run=False)
backup.update_restore_log(master, row_id, {'zookeeper': 'OK'})
else:
log.info('add_to_zk is not set, therefore not adding to zk')
except Exception as e:
log.warning("An exception occurred: {e}".format(e=e))
log.warning("If this is a DB issue, that's fine. "
"Otherwise, you should check ZK.")
backup.update_restore_log(master, row_id, {'finished_at': True})
log.info('Starting a new backup')
mysql_backup.mysql_backup(destination)