def rename_db_to_drop(instance, dbs, verbose=False, dry_run=False):
""" Create a new empty db and move the contents of the original db there
Args:
instance - a hostaddr object
dbs - a set of database names
verbose - bool, will direct sql to stdout
dry_run - bool, will make no changes to
"""
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
if not orphaned:
print "Detected no orphans"
sys.exit(1)
instance_orphans = orphaned[instance.__str__()]
unexpected = dbs.difference(instance_orphans)
if unexpected:
print ''.join(("Cowardly refusing to act on the following dbs: ",
','.join(unexpected)))
sys.exit(1)
# confirm that renames would not be blocked by an existing table
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
renamed_db = ''.join((DB_PREPEND, db))
sql = ''.join(("SELECT CONCAT(t2.TABLE_SCHEMA, \n",
" '.', t2.TABLE_NAME) as tbl \n",
"FROM information_schema.tables t1 \n",
"INNER JOIN information_schema.tables t2 \n",
" USING(TABLE_NAME) \n",
"WHERE t1.TABLE_SCHEMA = %(old_db)s AND \n"
" t2.TABLE_SCHEMA = %(new_db)s;"))
params = {'old_db': db,
'new_db': renamed_db}
cursor = conn.cursor()
cursor.execute(sql, params)
dups = cursor.fetchall()
if dups:
for dup in dups:
print "Table rename blocked by {tbl}".format(tbl=dup['tbl'])
sys.exit(1)
# We should be safe to create the new db and rename
if not dry_run:
mysql_lib.create_db(instance, renamed_db)
mysql_lib.move_db_contents(instance,
old_db=db,
new_db=renamed_db,
verbose=verbose,
dry_run=dry_run)
def rename_db_to_drop(instance, dbs, verbose=False, dry_run=False):
""" Create a new empty db and move the contents of the original db there
Args:
instance - a hostaddr object
dbs - a set of database names
verbose - bool, will direct sql to stdout
dry_run - bool, will make no changes to
"""
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
if not orphaned:
print "Detected no orphans"
sys.exit(1)
instance_orphans = orphaned[instance.__str__()]
unexpected = dbs.difference(instance_orphans)
if unexpected:
print ''.join(("Cowardly refusing to act on the following dbs: ",
','.join(unexpected)))
sys.exit(1)
# confirm that renames would not be blocked by an existing table
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
renamed_db = ''.join((DB_PREPEND, db))
sql = ''.join(("SELECT CONCAT(t2.TABLE_SCHEMA, \n",
" '.', t2.TABLE_NAME) as tbl \n",
"FROM information_schema.tables t1 \n",
"INNER JOIN information_schema.tables t2 \n",
" USING(TABLE_NAME) \n",
"WHERE t1.TABLE_SCHEMA = %(old_db)s AND \n"
" t2.TABLE_SCHEMA = %(new_db)s;"))
params = {'old_db': db,
'new_db': renamed_db}
cursor = conn.cursor()
cursor.execute(sql, params)
dups = cursor.fetchall()
if dups:
for dup in dups:
print "Table rename blocked by {tbl}".format(tbl=dup['tbl'])
sys.exit(1)
# We should be safe to create the new db and rename
if not dry_run:
mysql_lib.create_db(conn, renamed_db)
mysql_lib.move_db_contents(conn=conn,
old_db=db,
new_db=renamed_db,
verbose=verbose,
dry_run=dry_run)
def precheck_schema(instance):
""" Make sure the existing state is sane
Args:
instance - a hostAddr instance
"""
orphaned, orphaned_but_used, missing = \
find_shard_mismatches.find_shard_mismatches(instance)
if (orphaned or orphaned_but_used):
raise Exception('Unexpected shards are on {inst}. You can try to '
'clean them up using: '
'/usr/local/bin/mysql_utils/fix_orphaned_shards.py '
'-a rename -i {inst}'
''.format(inst=instance))
if missing:
raise Exception('Shards are missing on {}. This is really weird '
'and needs to be debugged'.format(instance))
def clean_up_migration(source_replica_set):
migration = start_shard_migration.check_migration_lock(source_replica_set)
destination_replica_set = migration['destination_replica_set']
mig_lock_identifier = migration['lock_identifier']
zk = host_utils.MysqlZookeeper()
destination_master = zk.get_mysql_instance_from_replica_set(
destination_replica_set)
try:
mysql_lib.get_slave_status(destination_master)
reset_repl = True
except:
reset_repl = False
if reset_repl:
log.info('Taking promotion locks')
dest_lock_identifier = mysql_failover.get_promotion_lock(
destination_replica_set)
log.info('Removing replication from destination master {}'
''.format(destination_master))
try:
mysql_lib.reset_slave(destination_master)
except:
raise
finally:
mysql_failover.release_promotion_lock(dest_lock_identifier)
(orphans_tmp, orphaned_but_used_tmp, _) = \
find_shard_mismatches.find_shard_mismatches(destination_master)
orphans = orphans_tmp[destination_master] if \
destination_master in orphans_tmp else []
orphaned_but_used = orphaned_but_used_tmp[destination_master] if \
destination_master in orphaned_but_used_tmp else []
if orphaned_but_used:
log.info('Orphaned but used dbs: {}'.format(
', '.join(orphaned_but_used)))
raise Exception('Cowardly refusing to do anything')
if orphans:
log.info('Orphaned dbs: {}'.format(', '.join(orphans)))
fix_orphaned_shards.rename_db_to_drop(destination_master, orphans)
start_shard_migration.finish_migration_log(
mig_lock_identifier, start_shard_migration.STATUS_ABORTED)
def drop_db_after_rename(instance, dbs, verbose, dry_run):
""" Drop the original empty db and a non-empty rename db
Args:
instance - a hostaddr object
dbs - a set of database names
verbose - bool, will direct sql to stdout
dry_run - bool, will make no changes to
"""
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
instance_orphans = orphaned[instance.__str__()]
unexpected = dbs.difference(instance_orphans)
if unexpected:
print ''.join(("Cowardly refusing to act on the following dbs: ",
','.join(unexpected)))
sys.exit(1)
# make sure the original db is empty
for db in dbs:
if mysql_lib.get_tables(instance, db):
print ''.join(("Cowardly refusing to drop non-empty db:",
db))
sys.exit(1)
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
# we should be good to drop the old empty dbs
raw_sql = 'DROP DATABASE IF EXISTS `{db}`;'
sql = raw_sql.format(db=db)
if verbose:
print sql
if not dry_run:
cursor.execute(sql)
# and we should be ok to drop the non-empty 'dropme_' prepended db
renamed_db = ''.join((DB_PREPEND, db))
sql = raw_sql.format(db=renamed_db)
if verbose:
print sql
if not dry_run:
cursor.execute(sql)
def drop_db_after_rename(instance, dbs, verbose, dry_run):
""" Drop the original empty db and a non-empty rename db
Args:
instance - a hostaddr object
dbs - a set of database names
verbose - bool, will direct sql to stdout
dry_run - bool, will make no changes to
"""
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
instance_orphans = orphaned[instance.__str__()]
unexpected = dbs.difference(instance_orphans)
if unexpected:
print ''.join(("Cowardly refusing to act on the following dbs: ",
','.join(unexpected)))
sys.exit(1)
# make sure the original db is empty
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
if mysql_lib.get_tables(conn, db):
print ''.join(("Cowardly refusing to drop non-empty db:",
db))
sys.exit(1)
for db in dbs:
# we should be good to drop the old empty dbs
raw_sql = 'DROP DATABASE IF EXISTS `{db}`;'
sql = raw_sql.format(db=db)
if verbose:
print sql
if not dry_run:
cursor.execute(sql)
# and we should be ok to drop the non-empty 'dropme_' prepended db
renamed_db = ''.join((DB_PREPEND, db))
sql = raw_sql.format(db=renamed_db)
if verbose:
print sql
if not dry_run:
cursor.execute(sql)
def drop_db_after_rename(instance, dbs=None, dry_run=False):
""" Drop the original empty db and a non-empty rename db
Args:
instance - a hostaddr object
dbs - a set of database names
dry_run - bool, will make no changes to the servers
"""
if not dbs:
dbs = set()
for db in mysql_lib.get_dbs(instance):
if db.startswith(DB_PREPEND):
dbs.add(db[len(DB_PREPEND):])
# confirm db is not in zk and not in use
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
instance_orphans = orphaned[instance]
unexpected = dbs.difference(instance_orphans)
if unexpected:
raise Exception('Cowardly refusing to act on the following '
'dbs: {}'.format(unexpected))
# make sure the original db is empty
for db in dbs:
if mysql_lib.get_tables(instance, db):
raise Exception('Cowardly refusing to drop non-empty '
'db: {}'.format(db))
for db in dbs:
renamed_db = ''.join((DB_PREPEND, db))
if dry_run:
log.info('dry_run is enabled, not dropping '
'dbs: {db} {renamed}'.format(db=db, renamed=renamed_db))
else:
mysql_lib.drop_db(instance, db)
mysql_lib.drop_db(instance, renamed_db)
def rename_db_to_drop(instance, dbs=None, dry_run=False, skip_check=False):
""" Create a new empty db and move the contents of the original db
into it
Args:
instance - a hostaddr object
dbs - a set of database names
dry_run - bool, will make no changes to anything
skip_check - Do not verify that db is not in production
"""
orphaned, _, _ = find_shard_mismatches.find_shard_mismatches(instance)
if not dbs:
if instance not in orphaned:
log.info("No orphaned shards, returning now.")
return
dbs = orphaned[instance]
log.info('Detected orphaned shareds: {}'.format(dbs))
if not skip_check:
# confirm db is not in ZK and not in use.
if not orphaned:
log.info("No orphans detected, returning now.")
return
instance_orphans = orphaned[instance]
unexpected = dbs.difference(instance_orphans)
if unexpected:
raise Exception('Cowardly refusing to act on the following'
'dbs: {}'.format(unexpected))
# confirm that renames would not be blocked by an existing table
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
for db in dbs:
# already dealt with
if db.startswith(DB_PREPEND):
continue
renamed_db = ''.join((DB_PREPEND, db))
sql = ''.join(("SELECT CONCAT(t2.TABLE_SCHEMA, \n",
" '.', t2.TABLE_NAME) as tbl \n",
"FROM information_schema.tables t1 \n",
"INNER JOIN information_schema.tables t2 \n",
" USING(TABLE_NAME) \n",
"WHERE t1.TABLE_SCHEMA = %(old_db)s AND \n"
" t2.TABLE_SCHEMA = %(new_db)s;"))
params = {'old_db': db, 'new_db': renamed_db}
cursor = conn.cursor()
cursor.execute(sql, params)
dups = cursor.fetchall()
if dups:
for dup in dups:
log.error('Table rename blocked by {}'.format(dup['tbl']))
sys.exit(1)
# We should be safe to create the new db and rename
if not dry_run:
mysql_lib.create_db(instance, renamed_db)
mysql_lib.move_db_contents(instance,
old_db=db,
new_db=renamed_db,
dry_run=dry_run)
if dbs and not dry_run:
log.info('To finish cleanup, wait a bit and then run:')
log.info('/usr/local/bin/mysql_utils/fix_orphaned_shards.py -a'
'drop -i {}'.format(instance))
