def process_mysql_shutdown(hostname=None, dry_run=False):
""" Check stats, and shutdown MySQL instances"""
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL)
if hostname:
if hostname in shutdown_instances:
log.info('Only acting on {}'.format(hostname))
shutdown_instances = {hostname: shutdown_instances[hostname]}
else:
log.info('Supplied host {} is not ready '
'for shutdown'.format(hostname))
return
for instance in shutdown_instances:
if instance in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for active_instance in zk.get_all_mysql_instances():
if active_instance.hostname == instance:
log.warning("It appears {instance} is in zk. This is "
"very dangerous! If you got to here, you may be "
"trying to turn down a replica set. Please remove "
"it from zk and try again"
"".format(instance=instance))
continue
if dry_run:
log.info('In dry_run mode, not changing state')
continue
try:
if check_for_user_activity(shutdown_instances[instance]):
log.info('Activity detected on {}, removing from queue'
''.format(instance))
remove_from_retirement_queue(hostname)
continue
else:
log.info('Shutting down mysql on {}'.format(instance))
mysql_lib.shutdown_mysql(host_utils.HostAddr(instance))
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.warning("Can't connect to MySQL on {}".format(instance))
log_to_retirement_queue(instance,
shutdown_instances[instance]['instance_id'],
SHUTDOWN_MYSQL)
def process_mysql_shutdown(hostname=None, dry_run=False):
""" Check stats, and shutdown MySQL instances"""
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL)
if hostname:
if hostname in shutdown_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
shutdown_instances = {hostname: shutdown_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for shutdown'.format(hostname=hostname))
return
for instance in shutdown_instances:
if instance in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for active_instance in zk.get_all_mysql_instances():
if active_instance.hostname == instance:
log.warning("It appears {instance} is in zk. This is "
"very dangerous! If you got to here, you may be "
"trying to turn down a replica set. Please remove "
"it from zk and try again"
"".format(instance=instance))
continue
# check mysql activity
if check_for_user_activity(shutdown_instances[instance]):
continue
# joining on a blank string as password must not have a space between
# the flag and the arg
if dry_run:
log.info('In dry_run mode, not changing state')
else:
log.info('Shuting down mysql on {instance}'.format(
instance=instance))
mysql_lib.shutdown_mysql(host_utils.HostAddr(instance))
log_to_retirement_queue(instance,
shutdown_instances[
instance]['instance_id'],
SHUTDOWN_MYSQL)
def process_mysql_shutdown(hostname=None, dry_run=False):
""" Check stats, and shutdown MySQL instances"""
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL)
if hostname:
if hostname in shutdown_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
shutdown_instances = {hostname: shutdown_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for shutdown'.format(hostname=hostname))
return
for instance in shutdown_instances:
if instance in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for active_instance in zk.get_all_mysql_instances():
if active_instance.hostname == instance:
log.warning("It appears {instance} is in zk. This is "
"very dangerous! If you got to here, you may be "
"trying to turn down a replica set. Please remove "
"it from zk and try again"
"".format(instance=instance))
continue
# check mysql activity
if check_for_user_activity(shutdown_instances[instance]):
continue
# joining on a blank string as password must not have a space between
# the flag and the arg
if dry_run:
log.info('In dry_run mode, not changing state')
else:
log.info(
'Shuting down mysql on {instance}'.format(instance=instance))
mysql_lib.shutdown_mysql(host_utils.HostAddr(instance))
log_to_retirement_queue(
instance, shutdown_instances[instance]['instance_id'],
SHUTDOWN_MYSQL)
def mysql_failover(master, dry_run, skip_lock,
ignore_dr_slave, trust_me_its_dead, kill_old_master):
""" Promte a new MySQL master
Args:
master - Hostaddr object of the master instance to be demoted
dry_run - Do not change state, just do sanity testing and exit
skip_lock - Do not take a promotion lock
ignore_dr_slave - Ignore the existance of a dr_slave
trust_me_its_dead - Do not test to see if the master is dead
kill_old_master - Send a mysqladmin kill command to the old master
Returns:
new_master - The new master server
"""
log.info('Master to demote is {master}'.format(master=master))
zk = host_utils.MysqlZookeeper()
(replica_set, _) = zk.get_replica_set_from_instance(master, rtypes=['master'])
log.info('Replica set is detected as '
'{replica_set}'.format(replica_set=replica_set))
# take a lock here to make sure nothing changes underneath us
if not skip_lock and not dry_run:
log.info('Taking promotion lock on replica set')
lock_identifier = get_promotion_lock(replica_set)
else:
lock_identifier = None
# giant try. If there any problems we roll back from the except
try:
master_conn = False
slave = zk.get_mysql_instance_from_replica_set(replica_set=replica_set,
repl_type=host_utils.REPLICA_ROLE_SLAVE)
log.info('Slave/new master is detected as {slave}'.format(slave=slave))
if ignore_dr_slave:
log.info('Intentionally ignoring a dr_slave')
dr_slave = None
else:
dr_slave = zk.get_mysql_instance_from_replica_set(replica_set,
host_utils.REPLICA_ROLE_DR_SLAVE)
log.info('DR slave is detected as {dr_slave}'.format(dr_slave=dr_slave))
if dr_slave:
if dr_slave == slave:
raise Exception('Slave and dr_slave appear to be the same')
replicas = set([slave, dr_slave])
else:
replicas = set([slave])
# let's make sure that what we think is the master, actually is
confirm_replica_topology(master, replicas)
# We use master_conn as a mysql connection to the master server, if
# it is False, the master is dead
if trust_me_its_dead:
master_conn = None
else:
master_conn = is_master_alive(master, replicas)
slave_conn = mysql_lib.connect_mysql(slave)
# Test to see if the slave is setup for replication. If not, we are hosed
log.info('Testing to see if Slave/new master is setup to write '
'replication logs')
try:
mysql_lib.get_master_status(slave_conn)
except mysql_lib.ReplicationError:
log.error('New master {slave} is not setup to write replicaiton '
'logs!'.format(slave=slave))
raise
log.info('Slave/new master is setup to write replication logs')
if kill_old_master:
log.info('Killing old master, we hope you know what you are doing')
mysql_lib.shutdown_mysql(master)
master_conn = None
if master_conn:
log.info('Master is considered alive')
dead_master = False
confirm_max_replica_lag(replicas, MAX_ALIVE_MASTER_SLAVE_LAG_SECONDS,
dead_master=dead_master)
else:
log.info('Master is considered dead')
dead_master = True
confirm_max_replica_lag(replicas, MAX_DEAD_MASTER_SLAVE_LAG_SECONDS,
dead_master=dead_master)
if dry_run:
log.info('In dry_run mode, so exiting now')
# Using os._exit in order to not get catch in the giant try
os._exit(0)
log.info('Preliminary sanity checks complete, starting promotion')
if master_conn:
log.info('Setting read_only on master')
mysql_lib.set_global_variable(master_conn, 'read_only', True)
log.info('Confirming no writes to old master')
# If there are writes with the master in read_only mode then the
# promotion can not proceed.
# A likely reason is a client has the SUPER privilege.
confirm_no_writes(master_conn)
log.info('Waiting for replicas to be caught up')
confirm_max_replica_lag(replicas, 0,
timeout=MAX_ALIVE_MASTER_SLAVE_LAG_SECONDS,
dead_master=dead_master)
log.info('Setting up replication from old master ({master})'
'to new master ({slave})'.format(master=master,
slave=slave))
mysql_lib.setup_replication(new_master=slave, new_replica=master)
else:
log.info('Starting up a zk connection to make sure we can connect')
kazoo_client = environment_specific.get_kazoo_client()
if not kazoo_client:
raise Exception('Could not conect to zk')
log.info('Confirming replica has processed all replication '
' logs')
confirm_no_writes(slave_conn)
log.info('Looks like no writes being processed by replica via '
'replication or other means')
if len(replicas) > 1:
log.info('Confirming relpica servers in sync')
confirm_max_replica_lag(replicas, MAX_DEAD_MASTER_SLAVE_LAG_SECONDS,
replicas_synced=True,
dead_master=dead_master)
except:
log.info('Starting rollback')
if master_conn:
log.info('Releasing read_only on old master')
mysql_lib.set_global_variable(master_conn, 'read_only', False)
log.info('Clearing replication settings on old master')
mysql_lib.reset_slave(master_conn)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Rollback complete, reraising exception')
raise
if dr_slave:
try:
mysql_lib.setup_replication(new_master=slave, new_replica=dr_slave)
except Exception as e:
log.error(e)
log.error('Setting up replication on the dr_slave failed. '
'Failing forward!')
log.info('Updating zk')
zk_write_attempt = 0
while True:
try:
modify_mysql_zk.swap_master_and_slave(slave, dry_run=False)
break
except:
if zk_write_attempt > MAX_ZK_WRITE_ATTEMPTS:
log.info('Final failure writing to zk, bailing')
raise
else:
log.info('Write to zk failed, trying again')
zk_write_attempt = zk_write_attempt+1
log.info('Removing read_only from new master')
mysql_lib.set_global_variable(slave_conn, 'read_only', False)
log.info('Removing replication configuration from new master')
mysql_lib.reset_slave(slave_conn)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Failover complete')
if not master_conn:
log.info('As master is dead, will try to launch a replacement. Will '
'sleep 20 seconds first to let things settle')
time.sleep(20)
launch_replacement_db_host.launch_replacement_db_host(master)
def mysql_failover(master, dry_run, skip_lock, ignore_dr_slave,
trust_me_its_dead, kill_old_master):
""" Promote a new MySQL master
Args:
master - Hostaddr object of the master instance to be demoted
dry_run - Do not change state, just do sanity testing and exit
skip_lock - Do not take a promotion lock
ignore_dr_slave - Ignore the existance of a dr_slave
trust_me_its_dead - Do not test to see if the master is dead
kill_old_master - Send a mysqladmin kill command to the old master
Returns:
new_master - The new master server
"""
log.info('Master to demote is {master}'.format(master=master))
zk = host_utils.MysqlZookeeper()
(replica_set, _) = zk.get_replica_set_from_instance(master,
rtypes=['master'])
log.info('Replica set is detected as '
'{replica_set}'.format(replica_set=replica_set))
# take a lock here to make sure nothing changes underneath us
if not skip_lock and not dry_run:
log.info('Taking promotion lock on replica set')
lock_identifier = get_promotion_lock(replica_set)
else:
lock_identifier = None
# giant try. If there any problems we roll back from the except
try:
master_conn = False
slave = zk.get_mysql_instance_from_replica_set(
replica_set=replica_set, repl_type=host_utils.REPLICA_ROLE_SLAVE)
log.info('Slave/new master is detected as {slave}'.format(slave=slave))
if ignore_dr_slave:
log.info('Intentionally ignoring a dr_slave')
dr_slave = None
else:
dr_slave = zk.get_mysql_instance_from_replica_set(
replica_set, host_utils.REPLICA_ROLE_DR_SLAVE)
log.info(
'DR slave is detected as {dr_slave}'.format(dr_slave=dr_slave))
if dr_slave:
if dr_slave == slave:
raise Exception('Slave and dr_slave appear to be the same')
replicas = set([slave, dr_slave])
else:
replicas = set([slave])
# We use master_conn as a mysql connection to the master server, if
# it is False, the master is dead
if trust_me_its_dead:
master_conn = None
else:
master_conn = is_master_alive(master, replicas)
# Test to see if the slave is setup for replication. If not, we are hosed
log.info('Testing to see if Slave/new master is setup to write '
'replication logs')
mysql_lib.get_master_status(slave)
if kill_old_master and not dry_run:
log.info('Killing old master, we hope you know what you are doing')
mysql_lib.shutdown_mysql(master)
master_conn = None
if master_conn:
log.info('Master is considered alive')
dead_master = False
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_NORMAL,
dead_master)
else:
log.info('Master is considered dead')
dead_master = True
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_LOOSE,
dead_master)
if dry_run:
log.info('In dry_run mode, so exiting now')
# Using os._exit in order to not get catch in the giant try
os._exit(environment_specific.DRY_RUN_EXIT_CODE)
log.info('Preliminary sanity checks complete, starting promotion')
if master_conn:
log.info('Setting read_only on master')
mysql_lib.set_global_variable(master, 'read_only', True)
log.info('Confirming no writes to old master')
# If there are writes with the master in read_only mode then the
# promotion can not proceed.
# A likely reason is a client has the SUPER privilege.
confirm_no_writes(master)
log.info('Waiting for replicas to be caught up')
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_NONE,
dead_master, True,
mysql_lib.NORMAL_HEARTBEAT_LAG)
log.info('Setting up replication from old master ({master}) '
'to new master ({slave})'.format(master=master,
slave=slave))
mysql_lib.setup_replication(new_master=slave, new_replica=master)
else:
log.info('Starting up a zk connection to make sure we can connect')
kazoo_client = environment_specific.get_kazoo_client()
if not kazoo_client:
raise Exception('Could not conect to zk')
log.info('Confirming replica has processed all replication '
' logs')
confirm_no_writes(slave)
log.info('Looks like no writes being processed by replica via '
'replication or other means')
if len(replicas) > 1:
log.info('Confirming replica servers are synced')
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_LOOSE,
dead_master, True)
except:
log.info('Starting rollback')
if master_conn:
log.info('Releasing read_only on old master')
mysql_lib.set_global_variable(master, 'read_only', False)
log.info('Clearing replication settings on old master')
mysql_lib.reset_slave(master)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Rollback complete, reraising exception')
raise
if dr_slave:
try:
mysql_lib.setup_replication(new_master=slave, new_replica=dr_slave)
except Exception as e:
log.error(e)
log.error('Setting up replication on the dr_slave failed. '
'Failing forward!')
log.info('Updating zk')
zk_write_attempt = 0
while True:
try:
modify_mysql_zk.swap_master_and_slave(slave, dry_run=False)
break
except:
if zk_write_attempt > MAX_ZK_WRITE_ATTEMPTS:
log.info('Final failure writing to zk, bailing')
raise
else:
log.info('Write to zk failed, trying again')
zk_write_attempt = zk_write_attempt + 1
log.info('Removing read_only from new master')
mysql_lib.set_global_variable(slave, 'read_only', False)
log.info('Removing replication configuration from new master')
mysql_lib.reset_slave(slave)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Failover complete')
# we don't really care if this fails, but we'll print a message anyway.
try:
environment_specific.generic_json_post(
environment_specific.CHANGE_FEED_URL, {
'type': 'MySQL Failover',
'environment': replica_set,
'description': "Failover from {m} to {s}".format(m=master,
s=slave),
'author': host_utils.get_user(),
'automation': False,
'source': "mysql_failover.py on {}".format(host_utils.HOSTNAME)
})
except Exception as e:
log.warning("Failover completed, but change feed "
"not updated: {}".format(e))
if not master_conn:
log.info('As master is dead, will try to launch a replacement. Will '
'sleep 20 seconds first to let things settle')
time.sleep(20)
launch_replacement_db_host.launch_replacement_db_host(master)
def mysql_failover(master, dry_run, skip_lock, ignore_dr_slave,
trust_me_its_dead, kill_old_master):
""" Promte a new MySQL master
Args:
master - Hostaddr object of the master instance to be demoted
dry_run - Do not change state, just do sanity testing and exit
skip_lock - Do not take a promotion lock
ignore_dr_slave - Ignore the existance of a dr_slave
trust_me_its_dead - Do not test to see if the master is dead
kill_old_master - Send a mysqladmin kill command to the old master
Returns:
new_master - The new master server
"""
log.info('Master to demote is {master}'.format(master=master))
zk = host_utils.MysqlZookeeper()
(replica_set, _) = zk.get_replica_set_from_instance(master,
rtypes=['master'])
log.info('Replica set is detected as '
'{replica_set}'.format(replica_set=replica_set))
# take a lock here to make sure nothing changes underneath us
if not skip_lock and not dry_run:
log.info('Taking promotion lock on replica set')
lock_identifier = get_promotion_lock(replica_set)
else:
lock_identifier = None
# giant try. If there any problems we roll back from the except
try:
master_conn = False
slave = zk.get_mysql_instance_from_replica_set(
replica_set=replica_set, repl_type=host_utils.REPLICA_ROLE_SLAVE)
log.info('Slave/new master is detected as {slave}'.format(slave=slave))
if ignore_dr_slave:
log.info('Intentionally ignoring a dr_slave')
dr_slave = None
else:
dr_slave = zk.get_mysql_instance_from_replica_set(
replica_set, host_utils.REPLICA_ROLE_DR_SLAVE)
log.info(
'DR slave is detected as {dr_slave}'.format(dr_slave=dr_slave))
if dr_slave:
if dr_slave == slave:
raise Exception('Slave and dr_slave appear to be the same')
replicas = set([slave, dr_slave])
else:
replicas = set([slave])
# let's make sure that what we think is the master, actually is
confirm_replica_topology(master, replicas)
# We use master_conn as a mysql connection to the master server, if
# it is False, the master is dead
if trust_me_its_dead:
master_conn = None
else:
master_conn = is_master_alive(master, replicas)
slave_conn = mysql_lib.connect_mysql(slave)
# Test to see if the slave is setup for replication. If not, we are hosed
log.info('Testing to see if Slave/new master is setup to write '
'replication logs')
try:
mysql_lib.get_master_status(slave_conn)
except mysql_lib.ReplicationError:
log.error('New master {slave} is not setup to write replicaiton '
'logs!'.format(slave=slave))
raise
log.info('Slave/new master is setup to write replication logs')
if kill_old_master:
log.info('Killing old master, we hope you know what you are doing')
mysql_lib.shutdown_mysql(master)
master_conn = None
if master_conn:
log.info('Master is considered alive')
dead_master = False
confirm_max_replica_lag(replicas,
MAX_ALIVE_MASTER_SLAVE_LAG_SECONDS,
dead_master=dead_master)
else:
log.info('Master is considered dead')
dead_master = True
confirm_max_replica_lag(replicas,
MAX_DEAD_MASTER_SLAVE_LAG_SECONDS,
dead_master=dead_master)
if dry_run:
log.info('In dry_run mode, so exiting now')
# Using os._exit in order to not get catch in the giant try
os._exit(0)
log.info('Preliminary sanity checks complete, starting promotion')
if master_conn:
log.info('Setting read_only on master')
mysql_lib.set_global_variable(master_conn, 'read_only', True)
log.info('Confirming no writes to old master')
# If there are writes with the master in read_only mode then the
# promotion can not proceed.
# A likely reason is a client has the SUPER privilege.
confirm_no_writes(master_conn)
log.info('Waiting for replicas to be caught up')
confirm_max_replica_lag(replicas,
0,
timeout=MAX_ALIVE_MASTER_SLAVE_LAG_SECONDS,
dead_master=dead_master)
log.info('Setting up replication from old master ({master})'
'to new master ({slave})'.format(master=master,
slave=slave))
mysql_lib.setup_replication(new_master=slave, new_replica=master)
else:
log.info('Starting up a zk connection to make sure we can connect')
kazoo_client = environment_specific.get_kazoo_client()
if not kazoo_client:
raise Exception('Could not conect to zk')
log.info('Confirming replica has processed all replication '
' logs')
confirm_no_writes(slave_conn)
log.info('Looks like no writes being processed by replica via '
'replication or other means')
if len(replicas) > 1:
log.info('Confirming relpica servers in sync')
confirm_max_replica_lag(replicas,
MAX_DEAD_MASTER_SLAVE_LAG_SECONDS,
replicas_synced=True,
dead_master=dead_master)
except:
log.info('Starting rollback')
if master_conn:
log.info('Releasing read_only on old master')
mysql_lib.set_global_variable(master_conn, 'read_only', False)
log.info('Clearing replication settings on old master')
mysql_lib.reset_slave(master_conn)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Rollback complete, reraising exception')
raise
if dr_slave:
try:
mysql_lib.setup_replication(new_master=slave, new_replica=dr_slave)
except Exception as e:
log.error(e)
log.error('Setting up replication on the dr_slave failed. '
'Failing forward!')
log.info('Updating zk')
zk_write_attempt = 0
while True:
try:
modify_mysql_zk.swap_master_and_slave(slave, dry_run=False)
break
except:
if zk_write_attempt > MAX_ZK_WRITE_ATTEMPTS:
log.info('Final failure writing to zk, bailing')
raise
else:
log.info('Write to zk failed, trying again')
zk_write_attempt = zk_write_attempt + 1
log.info('Removing read_only from new master')
mysql_lib.set_global_variable(slave_conn, 'read_only', False)
log.info('Removing replication configuration from new master')
mysql_lib.reset_slave(slave_conn)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Failover complete')
if not master_conn:
log.info('As master is dead, will try to launch a replacement. Will '
'sleep 20 seconds first to let things settle')
time.sleep(20)
launch_replacement_db_host.launch_replacement_db_host(master)
def process_mysql_shutdown(hostname=None, dry_run=False):
""" Check stats, and shutdown MySQL instances"""
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL)
if hostname:
if hostname in shutdown_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
shutdown_instances = {hostname: shutdown_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for shutdown'.format(hostname=hostname))
return
for instance in shutdown_instances:
if instance in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for active_instance in zk.get_all_mysql_instances():
if active_instance.hostname == instance:
log.warning("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
remove_from_retirement_queue(instance)
continue
log.info('Checking activity on {instance}'.format(instance=instance))
# check mysql activity
with timeout.timeout(3):
conn = MySQLdb.connect(host=shutdown_instances[instance]['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=shutdown_instances[instance]['internal_ip']))
activity = mysql_lib.get_user_activity(conn)
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected acitivty on {instance} by user(s):'
'{unexpected}'.format(instance=instance,
unexpected=','.join(unexpected)))
continue
log.info('Checking current connections on '
'{instance}'.format(instance=instance))
connected_users = mysql_lib.get_connected_users(conn)
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance,
unexpected=','.join(unexpected)))
continue
# joining on a blank string as password must not have a space between
# the flag and the arg
if dry_run:
log.info('In dry_run mode, not changing state')
else:
log.info('Shuting down mysql on {instance}'.format(instance=instance))
mysql_lib.shutdown_mysql(host_utils.HostAddr(instance))
log_to_retirement_queue(instance,
shutdown_instances[instance]['instance_id'],
SHUTDOWN_MYSQL)
def mysql_failover(master, dry_run, skip_lock,
trust_me_its_dead, kill_old_master):
""" Promote a new MySQL master
Args:
master - Hostaddr object of the master instance to be demoted
dry_run - Do not change state, just do sanity testing and exit
skip_lock - Do not take a promotion lock
trust_me_its_dead - Do not test to see if the master is dead
kill_old_master - Send a mysqladmin kill command to the old master
Returns:
new_master - The new master server
"""
log.info('Master to demote is {master}'.format(master=master))
slaves = host_utils.get_slaves_of_master(master)
if not slaves:
raise Exception('No slaves found.')
new_master = slaves[0]
replicas = slaves[1:]
# take a lock here to make sure nothing changes underneath us
if not skip_lock and not dry_run:
log.info('Taking promotion lock on {new_master}'.format(new_master=new_master))
cluster = host_utils.get_cluster_name(new_master)
lock_identifier = get_promotion_lock(cluster)
else:
lock_identifier = None
# giant try. If there any problems we roll back from the except
try:
master_conn = False
log.info('New master is detected as {slave}'.format(slave=new_master))
# We use master_conn as a mysql connection to the master server, if
# it is False, the master is dead
if trust_me_its_dead:
master_conn = None
else:
master_conn = is_master_alive(master, slaves)
# Test to see if the slave is setup for replication. If not, we are hosed
log.info('Testing to see if New master {new_master} is setup to write '
'replication logs'.format(new_master=new_master))
mysql_lib.get_master_status(new_master)
if kill_old_master and not dry_run:
log.info('Killing old master {}, we hope you know what you are doing'.format(master))
mysql_lib.shutdown_mysql(master)
master_conn = None
if master_conn:
log.info('Master {} is considered alive'.format(master))
dead_master = False
confirm_max_replica_lag(slaves,
mysql_lib.REPLICATION_TOLERANCE_NORMAL,
dead_master)
else:
log.info('Master {} is considered dead'.format(master))
dead_master = True
confirm_max_replica_lag(slaves,
mysql_lib.REPLICATION_TOLERANCE_LOOSE,
dead_master)
if dry_run:
log.info('In dry_run mode, so exiting now')
# Using os._exit in order to not get catch in the giant try
os._exit(0)
log.info('Preliminary sanity checks complete, starting promotion')
if master_conn:
log.info('Setting read_only on master {}'.format(master))
mysql_lib.set_global_read_only(master)
log.info('Confirming no writes to old master {}'.format(master))
# If there are writes with the master in read_only mode then the
# promotion can not proceed.
# A likely reason is a client has the SUPER privilege.
confirm_no_writes(master)
log.info('Waiting for replicas to be caught up')
confirm_max_replica_lag(slaves,
mysql_lib.REPLICATION_TOLERANCE_NONE,
dead_master,
True,
mysql_lib.NORMAL_HEARTBEAT_LAG)
log.info('Setting up replication from old master ({master}) '
'to new master ({slave})'.format(master=master,
slave=new_master))
mysql_lib.setup_replication(new_master=new_master, new_replica=master)
else:
log.info('Confirming replica/new master {} has processed all replication '
' logs'.format(new_master))
confirm_no_writes(new_master)
log.info('Looks like no writes being processed by replica {} via '
'replication or other means'.format(new_master))
if len(slaves) > 1:
log.info('Confirming replica servers are synced')
confirm_max_replica_lag(slaves,
mysql_lib.REPLICATION_TOLERANCE_LOOSE,
dead_master,
True)
except:
log.info('Starting rollback')
try:
if master_conn:
log.info('Releasing read_only on old master {}'.format(master))
mysql_lib.unset_global_read_only(master)
log.info('Clearing replication settings on old master {}'.format(master))
mysql_lib.reset_slave(master)
finally:
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Rollback complete, reraising exception')
raise
failed_replicas = []
for s in replicas:
log.info('Setting up replication {replication} from old master ({master}) '
'to new master ({slave})'.format(master=master,
slave=new_master,
replication=s))
try:
mysql_lib.setup_replication(new_master=new_master, new_replica=s)
except Exception as e:
failed_replicas.append(s)
log.error(e)
log.error('Setting up replication on the slave {} failed. Failing forward!'.format(s))
log.info('Updating config')
host_utils.swap_master_and_slave(new_master)
log.info('Removing read_only from new master {}'.format(new_master))
mysql_lib.unset_global_read_only(new_master)
log.info('Removing replication configuration from new master {}'.format(new_master))
mysql_lib.reset_slave(new_master)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
report(master, slaves, new_master, replicas, failed_replicas)
def mysql_failover(master, dry_run, skip_lock,
ignore_dr_slave, trust_me_its_dead, kill_old_master):
""" Promote a new MySQL master
Args:
master - Hostaddr object of the master instance to be demoted
dry_run - Do not change state, just do sanity testing and exit
skip_lock - Do not take a promotion lock
ignore_dr_slave - Ignore the existance of a dr_slave
trust_me_its_dead - Do not test to see if the master is dead
kill_old_master - Send a mysqladmin kill command to the old master
Returns:
new_master - The new master server
"""
log.info('Master to demote is {master}'.format(master=master))
zk = host_utils.MysqlZookeeper()
(replica_set, _) = zk.get_replica_set_from_instance(master, rtypes=['master'])
log.info('Replica set is detected as '
'{replica_set}'.format(replica_set=replica_set))
# take a lock here to make sure nothing changes underneath us
if not skip_lock and not dry_run:
log.info('Taking promotion lock on replica set')
lock_identifier = get_promotion_lock(replica_set)
else:
lock_identifier = None
# giant try. If there any problems we roll back from the except
try:
master_conn = False
slave = zk.get_mysql_instance_from_replica_set(replica_set=replica_set,
repl_type=host_utils.REPLICA_ROLE_SLAVE)
log.info('Slave/new master is detected as {slave}'.format(slave=slave))
if ignore_dr_slave:
log.info('Intentionally ignoring a dr_slave')
dr_slave = None
else:
dr_slave = zk.get_mysql_instance_from_replica_set(replica_set,
host_utils.REPLICA_ROLE_DR_SLAVE)
log.info('DR slave is detected as {dr_slave}'.format(dr_slave=dr_slave))
if dr_slave:
if dr_slave == slave:
raise Exception('Slave and dr_slave appear to be the same')
replicas = set([slave, dr_slave])
else:
replicas = set([slave])
# We use master_conn as a mysql connection to the master server, if
# it is False, the master is dead
if trust_me_its_dead:
master_conn = None
else:
master_conn = is_master_alive(master, replicas)
# Test to see if the slave is setup for replication. If not, we are hosed
log.info('Testing to see if Slave/new master is setup to write '
'replication logs')
mysql_lib.get_master_status(slave)
if kill_old_master and not dry_run:
log.info('Killing old master, we hope you know what you are doing')
mysql_lib.shutdown_mysql(master)
master_conn = None
if master_conn:
log.info('Master is considered alive')
dead_master = False
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_NORMAL,
dead_master)
else:
log.info('Master is considered dead')
dead_master = True
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_LOOSE,
dead_master)
if dry_run:
log.info('In dry_run mode, so exiting now')
# Using os._exit in order to not get catch in the giant try
os._exit(environment_specific.DRY_RUN_EXIT_CODE)
log.info('Preliminary sanity checks complete, starting promotion')
if master_conn:
log.info('Setting read_only on master')
mysql_lib.set_global_variable(master, 'read_only', True)
log.info('Confirming no writes to old master')
# If there are writes with the master in read_only mode then the
# promotion can not proceed.
# A likely reason is a client has the SUPER privilege.
confirm_no_writes(master)
log.info('Waiting for replicas to be caught up')
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_NONE,
dead_master,
True,
mysql_lib.NORMAL_HEARTBEAT_LAG)
log.info('Setting up replication from old master ({master}) '
'to new master ({slave})'.format(master=master,
slave=slave))
mysql_lib.setup_replication(new_master=slave, new_replica=master)
else:
log.info('Starting up a zk connection to make sure we can connect')
kazoo_client = environment_specific.get_kazoo_client()
if not kazoo_client:
raise Exception('Could not conect to zk')
log.info('Confirming replica has processed all replication '
' logs')
confirm_no_writes(slave)
log.info('Looks like no writes being processed by replica via '
'replication or other means')
if len(replicas) > 1:
log.info('Confirming replica servers are synced')
confirm_max_replica_lag(replicas,
mysql_lib.REPLICATION_TOLERANCE_LOOSE,
dead_master,
True)
except:
log.info('Starting rollback')
if master_conn:
log.info('Releasing read_only on old master')
mysql_lib.set_global_variable(master, 'read_only', False)
log.info('Clearing replication settings on old master')
mysql_lib.reset_slave(master)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Rollback complete, reraising exception')
raise
if dr_slave:
try:
mysql_lib.setup_replication(new_master=slave, new_replica=dr_slave)
except Exception as e:
log.error(e)
log.error('Setting up replication on the dr_slave failed. '
'Failing forward!')
log.info('Updating zk')
zk_write_attempt = 0
while True:
try:
modify_mysql_zk.swap_master_and_slave(slave, dry_run=False)
break
except:
if zk_write_attempt > MAX_ZK_WRITE_ATTEMPTS:
log.info('Final failure writing to zk, bailing')
raise
else:
log.info('Write to zk failed, trying again')
zk_write_attempt = zk_write_attempt+1
log.info('Removing read_only from new master')
mysql_lib.set_global_variable(slave, 'read_only', False)
log.info('Removing replication configuration from new master')
mysql_lib.reset_slave(slave)
if lock_identifier:
log.info('Releasing promotion lock')
release_promotion_lock(lock_identifier)
log.info('Failover complete')
# we don't really care if this fails, but we'll print a message anyway.
try:
environment_specific.generic_json_post(
environment_specific.CHANGE_FEED_URL,
{'type': 'MySQL Failover',
'environment': replica_set,
'description': "Failover from {m} to {s}".format(m=master, s=slave),
'author': host_utils.get_user(),
'automation': False,
'source': "mysql_failover.py on {}".format(host_utils.HOSTNAME)})
except Exception as e:
log.warning("Failover completed, but change feed "
"not updated: {}".format(e))
if not master_conn:
log.info('As master is dead, will try to launch a replacement. Will '
'sleep 20 seconds first to let things settle')
time.sleep(20)
launch_replacement_db_host.launch_replacement_db_host(master)