def manage_pt_heartbeat(instance): """ Restarts ptheartbeat if it isn't currently running and the replica role type is master, or stop it if it is running on a non-master. Args: instance (host_utils.HostAddr): host to check for ptheartbeat Returns: None """ connected_users = mysql_lib.get_connected_users(instance) zk = host_utils.MysqlZookeeper() try: replica_type = zk.get_replica_type_from_instance(instance) except: replica_type = None pthb_user, pthb_pass = mysql_lib.get_mysql_user_for_role('ptheartbeat') if replica_type == host_utils.REPLICA_ROLE_MASTER and \ pthb_user not in connected_users: host_utils.manage_pt_heartbeat(instance.port) log.info('Started process pt-heartbeat') elif replica_type != host_utils.REPLICA_ROLE_MASTER and \ pthb_user in connected_users: host_utils.manage_pt_heartbeat(instance.port, action='stop') log.info('Stopped pt-heartbeat on non-master replica')
def csv_backups_running(instance): """ Check to see if csv dumps are running Args: instance - we will use this to determine the replica set Returns: True if backups are running, False otherwise """ (dump_user, _) = mysql_lib.get_mysql_user_for_role(backup.USER_ROLE_MYSQLDUMP) replica_set = instance.get_zk_replica_set()[0] zk = host_utils.MysqlZookeeper() for slave_role in [ host_utils.REPLICA_ROLE_DR_SLAVE, host_utils.REPLICA_ROLE_SLAVE ]: slave_instance = zk.get_mysql_instance_from_replica_set( replica_set, slave_role) if not slave_instance: continue if dump_user in mysql_lib.get_connected_users(slave_instance): return True return False
def check_for_user_activity(instance): zk = host_utils.MysqlZookeeper() username, password = mysql_lib.get_mysql_user_for_role('admin') # check mysql activity log.info('Checking activity on {instance}'.format(instance=instance['hostname'])) with timeout.timeout(3): conn = MySQLdb.connect(host=instance['internal_ip'], user=username, passwd=password, cursorclass=MySQLdb.cursors.DictCursor) if not conn: raise Exception('Could not connect to {ip}' ''.format(ip=instance['internal_ip'])) activity = mysql_lib.get_user_activity(host_utils.HostAddr(instance['hostname'])) unexpected = set(activity.keys()).difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected activity on {instance} by user(s):' '{unexpected}'.format(instance=instance['hostname'], unexpected=','.join(unexpected))) return True log.info('Checking current connections on ' '{instance}'.format(instance=instance['hostname'])) connected_users = mysql_lib.get_connected_users(host_utils.HostAddr(instance['hostname'])) unexpected = connected_users.difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected connection on {instance} by user(s):' '{unexpected}'.format(instance=instance['hostname'], unexpected=','.join(unexpected))) return True return False
def restart_pt_kill_if_not_exists(instance): """ Restarts ptkill if it isn't currently running Args: instance (host_utils.HostAddr): host to check for ptkill Returns: None """ connected_users = mysql_lib.get_connected_users(instance) ptkill_user, ptkill_pass = mysql_lib.get_mysql_user_for_role('ptkill') if ptkill_user not in connected_users: host_utils.restart_pt_kill(instance.port) log.info('Started Processes ptkill')
def restart_pt_kill_if_not_exists(instance): """ Restarts ptkill if it isn't currently running Args: instance (host_utils.HostAddr): host to check for ptkill Returns: None """ connected_users = mysql_lib.get_connected_users(instance) ptkill_user, ptkill_pass = mysql_lib.get_mysql_user_for_role('ptkill') if ptkill_user not in connected_users: host_utils.manage_pt_kill(instance.port) log.info('Started Processes ptkill')
def restart_pt_heartbeat_if_not_exists(instance): """ Restarts ptheartbeat if it isn't currently running and the replica role type is master Args: instance (host_utils.HostAddr): host to check for ptheartbeat Returns: None """ connected_users = mysql_lib.get_connected_users(instance) zk = host_utils.MysqlZookeeper() try: (_, replica_type) = zk.get_replica_set_from_instance(instance) except: replica_type = None pthb_user, pthb_pass = mysql_lib.get_mysql_user_for_role('ptheartbeat') if replica_type in (host_utils.REPLICA_ROLE_MASTER, None) and \ pthb_user not in connected_users: host_utils.restart_pt_heartbeat(instance.port) log.info('Started Processes ptheartbeat')
def csv_backups_running(instance): """ Check to see if csv dumps are running Args: instance - we will use this to determine the replica set Returns: True if backups are running, False otherwise """ (dump_user, _) = mysql_lib.get_mysql_user_for_role(backup.USER_ROLE_MYSQLDUMP) replica_set = instance.get_zk_replica_set()[0] zk = host_utils.MysqlZookeeper() for slave_role in [host_utils.REPLICA_ROLE_DR_SLAVE, host_utils.REPLICA_ROLE_SLAVE]: slave_instance = zk.get_mysql_instance_from_replica_set(replica_set, slave_role) if not slave_instance: continue if dump_user in mysql_lib.get_connected_users(slave_instance): return True return False
def check_for_user_activity(instance): username, password = mysql_lib.get_mysql_user_for_role('admin') # check mysql activity log.info('Checking activity on {}'.format(instance['hostname'])) activity = mysql_lib.get_user_activity( host_utils.HostAddr(instance['hostname'])) unexpected = set(activity.keys()).difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected activity on {instance} by user(s):' '{unexpected}'.format(instance=instance['hostname'], unexpected=','.join(unexpected))) return True log.info('Checking current connections on ' '{instance}'.format(instance=instance['hostname'])) # try catch here due to the query creates the temp file will break our # code if disk space is full try: connected_users = mysql_lib.get_connected_users( host_utils.HostAddr(instance['hostname'])) except MySQLdb.InternalError as detail: (err_code, msg) = detail.args if err_code == mysql_lib.MYSQL_ERROR_CANT_CREATE_WRITE_TO_FILE: log.info('No space left on device') return False except: log.info('Something else is not correct here') return False unexpected = connected_users.difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected connection on {instance} by user(s):' '{unexpected}'.format(instance=instance['hostname'], unexpected=','.join(unexpected))) return True return False
def check_for_user_activity(instance): zk = host_utils.MysqlZookeeper() username, password = mysql_lib.get_mysql_user_for_role('admin') # check mysql activity log.info('Checking activity on {instance}'.format( instance=instance['hostname'])) with timeout.timeout(3): conn = MySQLdb.connect(host=instance['internal_ip'], user=username, passwd=password, cursorclass=MySQLdb.cursors.DictCursor) if not conn: raise Exception('Could not connect to {ip}' ''.format(ip=instance['internal_ip'])) activity = mysql_lib.get_user_activity( host_utils.HostAddr(instance['hostname'])) unexpected = set(activity.keys()).difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected activity on {instance} by user(s):' '{unexpected}'.format(instance=instance['hostname'], unexpected=','.join(unexpected))) return True log.info('Checking current connections on ' '{instance}'.format(instance=instance['hostname'])) connected_users = mysql_lib.get_connected_users( host_utils.HostAddr(instance['hostname'])) unexpected = connected_users.difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected connection on {instance} by user(s):' '{unexpected}'.format(instance=instance['hostname'], unexpected=','.join(unexpected))) return True return False
def process_mysql_shutdown(hostname=None, dry_run=False): """ Check stats, and shutdown MySQL instances""" zk = host_utils.MysqlZookeeper() username, password = mysql_lib.get_mysql_user_for_role('admin') shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL) if hostname: if hostname in shutdown_instances: log.info('Only acting on {hostname}'.format(hostname=hostname)) shutdown_instances = {hostname: shutdown_instances[hostname]} else: log.info('Supplied host {hostname} is not ready ' 'for shutdown'.format(hostname=hostname)) return for instance in shutdown_instances: if instance in get_protected_hosts('set'): log.warning('Host {hostname} is protected from ' 'retirement'.format(hostname=hostname)) remove_from_retirement_queue(hostname) continue for active_instance in zk.get_all_mysql_instances(): if active_instance.hostname == instance: log.warning("It appears {instance} is in zk. This is " "very dangerous!".format(instance=instance)) remove_from_retirement_queue(instance) continue log.info('Checking activity on {instance}'.format(instance=instance)) # check mysql activity with timeout.timeout(3): conn = MySQLdb.connect(host=shutdown_instances[instance]['internal_ip'], user=username, passwd=password, cursorclass=MySQLdb.cursors.DictCursor) if not conn: raise Exception('Could not connect to {ip}' ''.format(ip=shutdown_instances[instance]['internal_ip'])) activity = mysql_lib.get_user_activity(conn) unexpected = set(activity.keys()).difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected acitivty on {instance} by user(s):' '{unexpected}'.format(instance=instance, unexpected=','.join(unexpected))) continue log.info('Checking current connections on ' '{instance}'.format(instance=instance)) connected_users = mysql_lib.get_connected_users(conn) unexpected = connected_users.difference(IGNORABLE_USERS) if unexpected: log.error('Unexpected connection on {instance} by user(s):' '{unexpected}'.format(instance=instance, unexpected=','.join(unexpected))) continue # joining on a blank string as password must not have a space between # the flag and the arg if dry_run: log.info('In dry_run mode, not changing state') else: log.info('Shuting down mysql on {instance}'.format(instance=instance)) mysql_lib.shutdown_mysql(host_utils.HostAddr(instance)) log_to_retirement_queue(instance, shutdown_instances[instance]['instance_id'], SHUTDOWN_MYSQL)