def manage_pt_heartbeat(instance):
"""
Restarts ptheartbeat if it isn't currently running and the
replica role type is master, or stop it if it is running on
a non-master.
Args:
instance (host_utils.HostAddr): host to check for ptheartbeat
Returns:
None
"""
connected_users = mysql_lib.get_connected_users(instance)
zk = host_utils.MysqlZookeeper()
try:
replica_type = zk.get_replica_type_from_instance(instance)
except:
replica_type = None
pthb_user, pthb_pass = mysql_lib.get_mysql_user_for_role('ptheartbeat')
if replica_type == host_utils.REPLICA_ROLE_MASTER and \
pthb_user not in connected_users:
host_utils.manage_pt_heartbeat(instance.port)
log.info('Started process pt-heartbeat')
elif replica_type != host_utils.REPLICA_ROLE_MASTER and \
pthb_user in connected_users:
host_utils.manage_pt_heartbeat(instance.port, action='stop')
log.info('Stopped pt-heartbeat on non-master replica')
def csv_backups_running(instance):
""" Check to see if csv dumps are running
Args:
instance - we will use this to determine the replica set
Returns:
True if backups are running, False otherwise
"""
(dump_user,
_) = mysql_lib.get_mysql_user_for_role(backup.USER_ROLE_MYSQLDUMP)
replica_set = instance.get_zk_replica_set()[0]
zk = host_utils.MysqlZookeeper()
for slave_role in [
host_utils.REPLICA_ROLE_DR_SLAVE, host_utils.REPLICA_ROLE_SLAVE
]:
slave_instance = zk.get_mysql_instance_from_replica_set(
replica_set, slave_role)
if not slave_instance:
continue
if dump_user in mysql_lib.get_connected_users(slave_instance):
return True
return False
def check_for_user_activity(instance):
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
# check mysql activity
log.info('Checking activity on {instance}'.format(instance=instance['hostname']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=instance['internal_ip']))
activity = mysql_lib.get_user_activity(host_utils.HostAddr(instance['hostname']))
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected activity on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
log.info('Checking current connections on '
'{instance}'.format(instance=instance['hostname']))
connected_users = mysql_lib.get_connected_users(host_utils.HostAddr(instance['hostname']))
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
return False
def restart_pt_kill_if_not_exists(instance):
"""
Restarts ptkill if it isn't currently running
Args:
instance (host_utils.HostAddr): host to check for ptkill
Returns:
None
"""
connected_users = mysql_lib.get_connected_users(instance)
ptkill_user, ptkill_pass = mysql_lib.get_mysql_user_for_role('ptkill')
if ptkill_user not in connected_users:
host_utils.restart_pt_kill(instance.port)
log.info('Started Processes ptkill')
def restart_pt_kill_if_not_exists(instance):
"""
Restarts ptkill if it isn't currently running
Args:
instance (host_utils.HostAddr): host to check for ptkill
Returns:
None
"""
connected_users = mysql_lib.get_connected_users(instance)
ptkill_user, ptkill_pass = mysql_lib.get_mysql_user_for_role('ptkill')
if ptkill_user not in connected_users:
host_utils.manage_pt_kill(instance.port)
log.info('Started Processes ptkill')
def restart_pt_heartbeat_if_not_exists(instance):
"""
Restarts ptheartbeat if it isn't currently running and the
replica role type is master
Args:
instance (host_utils.HostAddr): host to check for ptheartbeat
Returns:
None
"""
connected_users = mysql_lib.get_connected_users(instance)
zk = host_utils.MysqlZookeeper()
try:
(_, replica_type) = zk.get_replica_set_from_instance(instance)
except:
replica_type = None
pthb_user, pthb_pass = mysql_lib.get_mysql_user_for_role('ptheartbeat')
if replica_type in (host_utils.REPLICA_ROLE_MASTER, None) and \
pthb_user not in connected_users:
host_utils.restart_pt_heartbeat(instance.port)
log.info('Started Processes ptheartbeat')
def restart_pt_heartbeat_if_not_exists(instance):
"""
Restarts ptheartbeat if it isn't currently running and the
replica role type is master
Args:
instance (host_utils.HostAddr): host to check for ptheartbeat
Returns:
None
"""
connected_users = mysql_lib.get_connected_users(instance)
zk = host_utils.MysqlZookeeper()
try:
(_, replica_type) = zk.get_replica_set_from_instance(instance)
except:
replica_type = None
pthb_user, pthb_pass = mysql_lib.get_mysql_user_for_role('ptheartbeat')
if replica_type in (host_utils.REPLICA_ROLE_MASTER, None) and \
pthb_user not in connected_users:
host_utils.restart_pt_heartbeat(instance.port)
log.info('Started Processes ptheartbeat')
def csv_backups_running(instance):
""" Check to see if csv dumps are running
Args:
instance - we will use this to determine the replica set
Returns:
True if backups are running, False otherwise
"""
(dump_user, _) = mysql_lib.get_mysql_user_for_role(backup.USER_ROLE_MYSQLDUMP)
replica_set = instance.get_zk_replica_set()[0]
zk = host_utils.MysqlZookeeper()
for slave_role in [host_utils.REPLICA_ROLE_DR_SLAVE, host_utils.REPLICA_ROLE_SLAVE]:
slave_instance = zk.get_mysql_instance_from_replica_set(replica_set, slave_role)
if not slave_instance:
continue
if dump_user in mysql_lib.get_connected_users(slave_instance):
return True
return False
def check_for_user_activity(instance):
username, password = mysql_lib.get_mysql_user_for_role('admin')
# check mysql activity
log.info('Checking activity on {}'.format(instance['hostname']))
activity = mysql_lib.get_user_activity(
host_utils.HostAddr(instance['hostname']))
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected activity on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
log.info('Checking current connections on '
'{instance}'.format(instance=instance['hostname']))
# try catch here due to the query creates the temp file will break our
# code if disk space is full
try:
connected_users = mysql_lib.get_connected_users(
host_utils.HostAddr(instance['hostname']))
except MySQLdb.InternalError as detail:
(err_code, msg) = detail.args
if err_code == mysql_lib.MYSQL_ERROR_CANT_CREATE_WRITE_TO_FILE:
log.info('No space left on device')
return False
except:
log.info('Something else is not correct here')
return False
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
return False
def check_for_user_activity(instance):
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
# check mysql activity
log.info('Checking activity on {instance}'.format(
instance=instance['hostname']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=instance['internal_ip']))
activity = mysql_lib.get_user_activity(
host_utils.HostAddr(instance['hostname']))
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected activity on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
log.info('Checking current connections on '
'{instance}'.format(instance=instance['hostname']))
connected_users = mysql_lib.get_connected_users(
host_utils.HostAddr(instance['hostname']))
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
return False
def process_mysql_shutdown(hostname=None, dry_run=False):
""" Check stats, and shutdown MySQL instances"""
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL)
if hostname:
if hostname in shutdown_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
shutdown_instances = {hostname: shutdown_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for shutdown'.format(hostname=hostname))
return
for instance in shutdown_instances:
if instance in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for active_instance in zk.get_all_mysql_instances():
if active_instance.hostname == instance:
log.warning("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
remove_from_retirement_queue(instance)
continue
log.info('Checking activity on {instance}'.format(instance=instance))
# check mysql activity
with timeout.timeout(3):
conn = MySQLdb.connect(host=shutdown_instances[instance]['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=shutdown_instances[instance]['internal_ip']))
activity = mysql_lib.get_user_activity(conn)
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected acitivty on {instance} by user(s):'
'{unexpected}'.format(instance=instance,
unexpected=','.join(unexpected)))
continue
log.info('Checking current connections on '
'{instance}'.format(instance=instance))
connected_users = mysql_lib.get_connected_users(conn)
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance,
unexpected=','.join(unexpected)))
continue
# joining on a blank string as password must not have a space between
# the flag and the arg
if dry_run:
log.info('In dry_run mode, not changing state')
else:
log.info('Shuting down mysql on {instance}'.format(instance=instance))
mysql_lib.shutdown_mysql(host_utils.HostAddr(instance))
log_to_retirement_queue(instance,
shutdown_instances[instance]['instance_id'],
SHUTDOWN_MYSQL)
