def add_to_queue(hostname, dry_run):
""" Add an instance to the retirement queue
Args:
hostname - The hostname of the instance to add to the retirement queue
"""
log.info('Adding server {hostname} to retirement '
'queue'.format(hostname=hostname))
if hostname in get_protected_hosts('set'):
raise Exception('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
# basic sanity check
zk = host_utils.MysqlZookeeper()
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
raise Exception("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
all_servers = environment_specific.get_all_server_metadata()
if not hostname in all_servers:
raise Exception('Host {hostname} is not cmdb'.format(hostname=hostname))
instance_metadata = all_servers[hostname]
log.info(instance_metadata)
username, password = mysql_lib.get_mysql_user_for_role('admin')
try:
log.info('Trying to reset user_statistics on ip '
'{ip}'.format(ip=instance_metadata['internal_ip']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance_metadata['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('timeout')
mysql_lib.enable_and_flush_activity_statistics(conn)
activity = RESET_STATS
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('Could not connect to '
'{ip}'.format(ip=instance_metadata['internal_ip']))
activity = SHUTDOWN_MYSQL
log_to_retirement_queue(hostname, instance_metadata['instance_id'],
activity)
def confirm_no_writes(conn):
""" Confirm that a server is not receiving any writes
Args:
conn - A mysql connection
"""
mysql_lib.enable_and_flush_activity_statistics(conn)
log.info('Waiting {length} seconds to confirm instance is no longer '
'accepting writes'.format(length=WAIT_TIME_CONFIRM_QUIESCE))
time.sleep(WAIT_TIME_CONFIRM_QUIESCE)
db_activity = mysql_lib.get_dbs_activity(conn)
active_db = set()
for db in db_activity:
if db_activity[db]['ROWS_CHANGED'] != 0:
active_db.add(db)
if active_db:
raise Exception('DB {dbs} has been modified when it should have '
'no activity'.format(dbs=active_db))
log.info('No writes after sleep, looks like we are good to go')
def confirm_no_writes(instance):
""" Confirm that a server is not receiving any writes
Args:
conn - A mysql connection
"""
mysql_lib.enable_and_flush_activity_statistics(instance)
log.info('Waiting {length} seconds to confirm instance is no longer '
'accepting writes'.format(length=WAIT_TIME_CONFIRM_QUIESCE))
time.sleep(WAIT_TIME_CONFIRM_QUIESCE)
db_activity = mysql_lib.get_dbs_activity(instance)
active_db = set()
for db in db_activity:
if db_activity[db]['ROWS_CHANGED'] != 0:
active_db.add(db)
if active_db:
raise Exception('DB {dbs} has been modified when it should have '
'no activity'.format(dbs=active_db))
log.info('No writes after sleep, looks like we are good to go')
def add_to_queue(hostname, dry_run, skip_production_check=False):
""" Add an instance to the retirement queue
Args:
hostname - The hostname of the instance to add to the retirement queue
"""
log.info('Adding server {hostname} to retirement '
'queue'.format(hostname=hostname))
if hostname in get_protected_hosts('set'):
raise Exception('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
# basic sanity check
zk = host_utils.MysqlZookeeper()
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
if skip_production_check:
log.warning("It appears {instance} is in zk but "
"skip_production_check is set so continuing."
"".format(instance=instance))
else:
raise Exception("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
all_servers = environment_specific.get_all_server_metadata()
if hostname not in all_servers:
raise Exception(
'Host {hostname} is not cmdb'.format(hostname=hostname))
instance_metadata = all_servers[hostname]
log.info(instance_metadata)
username, password = mysql_lib.get_mysql_user_for_role('admin')
try:
if check_for_user_activity(instance_metadata):
log.info('Trying to reset user_statistics on ip '
'{ip}'.format(ip=instance_metadata['internal_ip']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance_metadata['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('timeout')
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
mysql_lib.enable_and_flush_activity_statistics(
host_utils.HostAddr(hostname))
else:
log.info("No recent user activity, skipping stats reset")
# We still need to add it to the queue the first time.
# Check if it was added recently and exit if it was
if is_host_in_retirement_queue(hostname):
return
activity = RESET_STATS
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('Could not connect to '
'{ip}'.format(ip=instance_metadata['internal_ip']))
activity = SHUTDOWN_MYSQL
# We only want to add the host if it wasn't already in the queue
if is_host_in_retirement_queue(hostname):
return
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
log_to_retirement_queue(hostname, instance_metadata['instance_id'],
activity)
def add_to_queue(hostname, dry_run, skip_production_check=False):
""" Add an instance to the retirement queue
Args:
hostname - The hostname of the instance to add to the retirement queue
"""
log.info('Adding server {hostname} to retirement '
'queue'.format(hostname=hostname))
if hostname in get_protected_hosts('set'):
raise Exception('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
# basic sanity check
zk = host_utils.MysqlZookeeper()
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
if skip_production_check:
log.warning("It appears {instance} is in zk but "
"skip_production_check is set so continuing."
"".format(instance=instance))
else:
raise Exception("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
all_servers = environment_specific.get_all_server_metadata()
if hostname not in all_servers:
raise Exception('Host {hostname} is not cmdb'.format(hostname=hostname))
instance_metadata = all_servers[hostname]
log.info(instance_metadata)
username, password = mysql_lib.get_mysql_user_for_role('admin')
try:
if check_for_user_activity(instance_metadata):
log.info('Trying to reset user_statistics on ip '
'{ip}'.format(ip=instance_metadata['internal_ip']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance_metadata['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('timeout')
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
mysql_lib.enable_and_flush_activity_statistics(host_utils.HostAddr(hostname))
else:
log.info("No recent user activity, skipping stats reset")
# We still need to add it to the queue the first time.
# Check if it was added recently and exit if it was
if is_host_in_retirement_queue(hostname):
return
activity = RESET_STATS
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('Could not connect to '
'{ip}'.format(ip=instance_metadata['internal_ip']))
activity = SHUTDOWN_MYSQL
# We only want to add the host if it wasn't already in the queue
if is_host_in_retirement_queue(hostname):
return
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
log_to_retirement_queue(hostname, instance_metadata['instance_id'],
activity)
