def rename_db_to_drop(instance, dbs, verbose=False, dry_run=False):
""" Create a new empty db and move the contents of the original db there
Args:
instance - a hostaddr object
dbs - a set of database names
verbose - bool, will direct sql to stdout
dry_run - bool, will make no changes to
"""
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
if not orphaned:
print "Detected no orphans"
sys.exit(1)
instance_orphans = orphaned[instance.__str__()]
unexpected = dbs.difference(instance_orphans)
if unexpected:
print ''.join(("Cowardly refusing to act on the following dbs: ",
','.join(unexpected)))
sys.exit(1)
# confirm that renames would not be blocked by an existing table
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
renamed_db = ''.join((DB_PREPEND, db))
sql = ''.join(("SELECT CONCAT(t2.TABLE_SCHEMA, \n",
" '.', t2.TABLE_NAME) as tbl \n",
"FROM information_schema.tables t1 \n",
"INNER JOIN information_schema.tables t2 \n",
" USING(TABLE_NAME) \n",
"WHERE t1.TABLE_SCHEMA = %(old_db)s AND \n"
" t2.TABLE_SCHEMA = %(new_db)s;"))
params = {'old_db': db,
'new_db': renamed_db}
cursor = conn.cursor()
cursor.execute(sql, params)
dups = cursor.fetchall()
if dups:
for dup in dups:
print "Table rename blocked by {tbl}".format(tbl=dup['tbl'])
sys.exit(1)
# We should be safe to create the new db and rename
if not dry_run:
mysql_lib.create_db(instance, renamed_db)
mysql_lib.move_db_contents(instance,
old_db=db,
new_db=renamed_db,
verbose=verbose,
dry_run=dry_run)
def rename_db_to_drop(instance, dbs, verbose=False, dry_run=False):
""" Create a new empty db and move the contents of the original db there
Args:
instance - a hostaddr object
dbs - a set of database names
verbose - bool, will direct sql to stdout
dry_run - bool, will make no changes to
"""
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
if not orphaned:
print "Detected no orphans"
sys.exit(1)
instance_orphans = orphaned[instance.__str__()]
unexpected = dbs.difference(instance_orphans)
if unexpected:
print ''.join(("Cowardly refusing to act on the following dbs: ",
','.join(unexpected)))
sys.exit(1)
# confirm that renames would not be blocked by an existing table
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
renamed_db = ''.join((DB_PREPEND, db))
sql = ''.join(("SELECT CONCAT(t2.TABLE_SCHEMA, \n",
" '.', t2.TABLE_NAME) as tbl \n",
"FROM information_schema.tables t1 \n",
"INNER JOIN information_schema.tables t2 \n",
" USING(TABLE_NAME) \n",
"WHERE t1.TABLE_SCHEMA = %(old_db)s AND \n"
" t2.TABLE_SCHEMA = %(new_db)s;"))
params = {'old_db': db,
'new_db': renamed_db}
cursor = conn.cursor()
cursor.execute(sql, params)
dups = cursor.fetchall()
if dups:
for dup in dups:
print "Table rename blocked by {tbl}".format(tbl=dup['tbl'])
sys.exit(1)
# We should be safe to create the new db and rename
if not dry_run:
mysql_lib.create_db(conn, renamed_db)
mysql_lib.move_db_contents(conn=conn,
old_db=db,
new_db=renamed_db,
verbose=verbose,
dry_run=dry_run)
def mysql_init_server(instance,
skip_production_check=False,
skip_backup=True,
lock_handle=None):
""" Remove any data and initialize a MySQL instance
Args:
instance - A hostaddr object pointing towards localhost to act upon
skip_production_check - Dangerous! will not run safety checks to protect
production data
skip_backup - Don't run a backup after the instance is setup
lock_handle - If the caller already locked the system, pass in the
lock handle, as we may need to release and reacquire
to prevent mysqld from keeping it.
"""
if lock_handle is None:
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a lock to block race conditions')
lock_handle = host_utils.bind_lock_socket(
backup.STD_BACKUP_LOCK_SOCKET)
else:
log.info('Lock already exists from caller.')
try:
# sanity check
zk = host_utils.MysqlZookeeper()
if (not skip_production_check
and instance in zk.get_all_mysql_instances()):
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=instance))
log.info('Checking host for mounts, etc...')
basic_host_sanity()
log.info('(re)Generating MySQL cnf files')
mysql_cnf_builder.build_cnf()
log.info('Creating any missing directories')
create_and_chown_dirs(instance.port)
log.info('Shutting down MySQL (if applicable)')
host_utils.stop_mysql(instance.port)
log.info('Deleting existing MySQL data')
delete_mysql_data(instance.port)
log.info('Creating MySQL privileges tables')
init_privileges_tables(instance.port)
log.info('Clearing innodb log files')
delete_innodb_log_files(instance.port)
log.info('Starting up instance')
host_utils.start_mysql(instance.port)
log.info('Importing MySQL users')
mysql_grants.manage_mysql_grants(instance, 'nuke_then_import')
log.info('Creating test database')
mysql_lib.create_db(instance, 'test')
log.info('Setting up query response time plugins')
mysql_lib.setup_response_time_metrics(instance)
log.info('Setting up semi-sync replication plugins')
mysql_lib.setup_semisync_plugins(instance)
log.info('Setting up audit log plugin')
mysql_lib.setup_audit_plugin(instance)
log.info('Restarting pt daemons')
host_utils.manage_pt_daemons(instance.port)
log.info('MySQL initalization complete')
finally:
# We have to do this, ugly though it may be, to ensure that
# the running MySQL process doesn't maintain a hold on the lock
# socket after the script exits. We reacquire the lock after
# the restart and pass it back to the caller.
#
if lock_handle:
log.info('Restarting MySQL, releasing lock.')
host_utils.stop_mysql(instance.port)
log.info('Sleeping 5 seconds.')
time.sleep(5)
host_utils.release_lock_socket(lock_handle)
host_utils.start_mysql(instance.port)
log.info('Reacquiring lock.')
lock_handle = host_utils.bind_lock_socket(
backup.STD_BACKUP_LOCK_SOCKET)
if not skip_backup:
log.info('Taking a backup')
mysql_backup.mysql_backup(instance,
initial_build=True,
lock_handle=lock_handle)
return lock_handle
def mysql_init_server(instance,
skip_production_check=False,
skip_locking=False,
skip_backup=True):
""" Remove any data and initialize a MySQL instance
Args:
instance - A hostaddr object pointing towards localhost to act upon
skip_production_check - Dangerous! will not run safety checks to protect
production data
skip_locking - Do not take a lock on localhost. Useful when the caller has
already has taken the lock (ie mysql_restore_xtrabackup)
skip_backup - Don't run a backup after the instance is setup
"""
lock_handle = None
if not skip_locking:
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block race conditions')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
try:
# sanity check
zk = host_utils.MysqlZookeeper()
if (not skip_production_check
and instance in zk.get_all_mysql_instances()):
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=instance))
log.info('Checking host for mounts, etc...')
basic_host_sanity()
log.info('(re)Generating MySQL cnf files')
mysql_cnf_builder.build_cnf()
log.info('Creating any missing directories')
create_and_chown_dirs(instance.port)
log.info('Shutting down MySQL (if applicable)')
host_utils.stop_mysql(instance.port)
log.info('Deleting existing MySQL data')
delete_mysql_data(instance.port)
log.info('Creating MySQL privileges tables')
init_privileges_tables(instance.port)
log.info('Clearing innodb log files')
delete_innodb_log_files(instance.port)
log.info('Starting up instance')
host_utils.start_mysql(instance.port)
log.info('Importing MySQL users')
mysql_grants.manage_mysql_grants(instance, 'nuke_then_import')
log.info('Creating test database')
mysql_lib.create_db(instance, 'test')
log.info('Setting up query response time plugins')
mysql_lib.setup_response_time_metrics(instance)
log.info('Setting up semi-sync replication plugins')
mysql_lib.setup_semisync_plugins(instance)
log.info('Restarting pt daemons')
host_utils.restart_pt_daemons(instance.port)
log.info('MySQL initalization complete')
finally:
if not skip_locking and lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
if not skip_backup:
log.info('Taking a backup')
mysql_backup.mysql_backup(instance)
def mysql_init_server(instance,
skip_production_check=False, skip_locking=False,
skip_backup=True):
""" Remove any data and initialize a MySQL instance
Args:
instance - A hostaddr object pointing towards localhost to act upon
skip_production_check - Dangerous! will not run safety checks to protect
production data
skip_locking - Do not take a lock on localhost. Useful when the caller has
already has taken the lock (ie mysql_restore_xtrabackup)
skip_backup - Don't run a backup after the instance is setup
"""
lock_handle = None
if not skip_locking:
# Take a lock to prevent multiple restores from running concurrently
log.info('Taking a flock to block race conditions')
lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)
try:
# sanity check
zk = host_utils.MysqlZookeeper()
if (not skip_production_check and
instance in zk.get_all_mysql_instances()):
raise Exception("It appears {instance} is in use. This is"
" very dangerous!".format(instance=instance))
log.info('Checking host for mounts, etc...')
basic_host_sanity()
log.info('(re)Generating MySQL cnf files')
mysql_cnf_builder.build_cnf()
log.info('Creating any missing directories')
create_and_chown_dirs(instance.port)
log.info('Shutting down MySQL (if applicable)')
host_utils.stop_mysql(instance.port)
log.info('Deleting existing MySQL data')
delete_mysql_data(instance.port)
log.info('Creating MySQL privileges tables')
init_privileges_tables(instance.port)
log.info('Clearing innodb log files')
delete_innodb_log_files(instance.port)
log.info('Starting up instance')
host_utils.start_mysql(instance.port)
log.info('Importing MySQL users')
mysql_grants.manage_mysql_grants(instance, 'nuke_then_import')
log.info('Creating test database')
conn = mysql_lib.connect_mysql(instance)
mysql_lib.create_db(conn, 'test')
log.info('Setting up query response time plugins')
mysql_lib.setup_response_time_metrics(instance)
log.info('Setting up semi-sync replication plugins')
mysql_lib.setup_semisync_plugins(instance)
log.info('Restarting pt daemons')
host_utils.restart_pt_daemons(instance.port)
log.info('MySQL initalization complete')
finally:
if not skip_locking and lock_handle:
log.info('Releasing lock')
host_utils.release_flock_lock(lock_handle)
if not skip_backup:
log.info('Taking a backup')
mysql_backup.mysql_backup(instance)
def rename_db_to_drop(instance, dbs=None, dry_run=False, skip_check=False):
""" Create a new empty db and move the contents of the original db
into it
Args:
instance - a hostaddr object
dbs - a set of database names
dry_run - bool, will make no changes to anything
skip_check - Do not verify that db is not in production
"""
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
if not dbs:
if instance not in orphaned:
log.info("No orphaned shards, returning now.")
return
dbs = orphaned[instance]
log.info('Detected orphaned shareds: {}'.format(dbs))
if not skip_check:
# confirm db is not in ZK and not in use.
if not orphaned:
log.info("No orphans detected, returning now.")
return
instance_orphans = orphaned[instance]
unexpected = dbs.difference(instance_orphans)
if unexpected:
raise Exception('Cowardly refusing to act on the following'
'dbs: {}'.format(unexpected))
# confirm that renames would not be blocked by an existing table
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
# already dealt with
if db.startswith(DB_PREPEND):
continue
renamed_db = ''.join((DB_PREPEND, db))
sql = ''.join(("SELECT CONCAT(t2.TABLE_SCHEMA, \n",
" '.', t2.TABLE_NAME) as tbl \n",
"FROM information_schema.tables t1 \n",
"INNER JOIN information_schema.tables t2 \n",
" USING(TABLE_NAME) \n",
"WHERE t1.TABLE_SCHEMA = %(old_db)s AND \n"
" t2.TABLE_SCHEMA = %(new_db)s;"))
params = {'old_db': db, 'new_db': renamed_db}
cursor = conn.cursor()
cursor.execute(sql, params)
dups = cursor.fetchall()
if dups:
for dup in dups:
log.error('Table rename blocked by {}'.format(dup['tbl']))
sys.exit(1)
# We should be safe to create the new db and rename
if not dry_run:
mysql_lib.create_db(instance, renamed_db)
mysql_lib.move_db_contents(instance,
old_db=db,
new_db=renamed_db,
dry_run=dry_run)
if dbs and not dry_run:
log.info('To finish cleanup, wait a bit and then run:')
log.info('/usr/local/bin/mysql_utils/fix_orphaned_shards.py -a'
'drop -i {}'.format(instance))