def add_to_queue(hostname, dry_run): """ Add an instance to the retirement queue Args: hostname - The hostname of the instance to add to the retirement queue """ log.info('Adding server {hostname} to retirement ' 'queue'.format(hostname=hostname)) if hostname in get_protected_hosts('set'): raise Exception('Host {hostname} is protected from ' 'retirement'.format(hostname=hostname)) # basic sanity check zk = host_utils.MysqlZookeeper() for instance in zk.get_all_mysql_instances(): if instance.hostname == hostname: raise Exception("It appears {instance} is in zk. This is " "very dangerous!".format(instance=instance)) all_servers = environment_specific.get_all_server_metadata() if not hostname in all_servers: raise Exception('Host {hostname} is not cmdb'.format(hostname=hostname)) instance_metadata = all_servers[hostname] log.info(instance_metadata) username, password = mysql_lib.get_mysql_user_for_role('admin') try: log.info('Trying to reset user_statistics on ip ' '{ip}'.format(ip=instance_metadata['internal_ip'])) with timeout.timeout(3): conn = MySQLdb.connect(host=instance_metadata['internal_ip'], user=username, passwd=password, cursorclass=MySQLdb.cursors.DictCursor) if not conn: raise Exception('timeout') mysql_lib.enable_and_flush_activity_statistics(conn) activity = RESET_STATS except MySQLdb.OperationalError as detail: (error_code, msg) = detail.args if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR: raise log.info('Could not connect to ' '{ip}'.format(ip=instance_metadata['internal_ip'])) activity = SHUTDOWN_MYSQL log_to_retirement_queue(hostname, instance_metadata['instance_id'], activity)
def confirm_no_writes(conn): """ Confirm that a server is not receiving any writes Args: conn - A mysql connection """ mysql_lib.enable_and_flush_activity_statistics(conn) log.info('Waiting {length} seconds to confirm instance is no longer ' 'accepting writes'.format(length=WAIT_TIME_CONFIRM_QUIESCE)) time.sleep(WAIT_TIME_CONFIRM_QUIESCE) db_activity = mysql_lib.get_dbs_activity(conn) active_db = set() for db in db_activity: if db_activity[db]['ROWS_CHANGED'] != 0: active_db.add(db) if active_db: raise Exception('DB {dbs} has been modified when it should have ' 'no activity'.format(dbs=active_db)) log.info('No writes after sleep, looks like we are good to go')
def confirm_no_writes(instance): """ Confirm that a server is not receiving any writes Args: conn - A mysql connection """ mysql_lib.enable_and_flush_activity_statistics(instance) log.info('Waiting {length} seconds to confirm instance is no longer ' 'accepting writes'.format(length=WAIT_TIME_CONFIRM_QUIESCE)) time.sleep(WAIT_TIME_CONFIRM_QUIESCE) db_activity = mysql_lib.get_dbs_activity(instance) active_db = set() for db in db_activity: if db_activity[db]['ROWS_CHANGED'] != 0: active_db.add(db) if active_db: raise Exception('DB {dbs} has been modified when it should have ' 'no activity'.format(dbs=active_db)) log.info('No writes after sleep, looks like we are good to go')
def add_to_queue(hostname, dry_run, skip_production_check=False): """ Add an instance to the retirement queue Args: hostname - The hostname of the instance to add to the retirement queue """ log.info('Adding server {hostname} to retirement ' 'queue'.format(hostname=hostname)) if hostname in get_protected_hosts('set'): raise Exception('Host {hostname} is protected from ' 'retirement'.format(hostname=hostname)) # basic sanity check zk = host_utils.MysqlZookeeper() for instance in zk.get_all_mysql_instances(): if instance.hostname == hostname: if skip_production_check: log.warning("It appears {instance} is in zk but " "skip_production_check is set so continuing." "".format(instance=instance)) else: raise Exception("It appears {instance} is in zk. This is " "very dangerous!".format(instance=instance)) all_servers = environment_specific.get_all_server_metadata() if hostname not in all_servers: raise Exception( 'Host {hostname} is not cmdb'.format(hostname=hostname)) instance_metadata = all_servers[hostname] log.info(instance_metadata) username, password = mysql_lib.get_mysql_user_for_role('admin') try: if check_for_user_activity(instance_metadata): log.info('Trying to reset user_statistics on ip ' '{ip}'.format(ip=instance_metadata['internal_ip'])) with timeout.timeout(3): conn = MySQLdb.connect(host=instance_metadata['internal_ip'], user=username, passwd=password, cursorclass=MySQLdb.cursors.DictCursor) if not conn: raise Exception('timeout') if dry_run: log.info('In dry_run mode, not changing anything') else: mysql_lib.enable_and_flush_activity_statistics( host_utils.HostAddr(hostname)) else: log.info("No recent user activity, skipping stats reset") # We still need to add it to the queue the first time. # Check if it was added recently and exit if it was if is_host_in_retirement_queue(hostname): return activity = RESET_STATS except MySQLdb.OperationalError as detail: (error_code, msg) = detail.args if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR: raise log.info('Could not connect to ' '{ip}'.format(ip=instance_metadata['internal_ip'])) activity = SHUTDOWN_MYSQL # We only want to add the host if it wasn't already in the queue if is_host_in_retirement_queue(hostname): return if dry_run: log.info('In dry_run mode, not changing anything') else: log_to_retirement_queue(hostname, instance_metadata['instance_id'], activity)
def add_to_queue(hostname, dry_run, skip_production_check=False): """ Add an instance to the retirement queue Args: hostname - The hostname of the instance to add to the retirement queue """ log.info('Adding server {hostname} to retirement ' 'queue'.format(hostname=hostname)) if hostname in get_protected_hosts('set'): raise Exception('Host {hostname} is protected from ' 'retirement'.format(hostname=hostname)) # basic sanity check zk = host_utils.MysqlZookeeper() for instance in zk.get_all_mysql_instances(): if instance.hostname == hostname: if skip_production_check: log.warning("It appears {instance} is in zk but " "skip_production_check is set so continuing." "".format(instance=instance)) else: raise Exception("It appears {instance} is in zk. This is " "very dangerous!".format(instance=instance)) all_servers = environment_specific.get_all_server_metadata() if hostname not in all_servers: raise Exception('Host {hostname} is not cmdb'.format(hostname=hostname)) instance_metadata = all_servers[hostname] log.info(instance_metadata) username, password = mysql_lib.get_mysql_user_for_role('admin') try: if check_for_user_activity(instance_metadata): log.info('Trying to reset user_statistics on ip ' '{ip}'.format(ip=instance_metadata['internal_ip'])) with timeout.timeout(3): conn = MySQLdb.connect(host=instance_metadata['internal_ip'], user=username, passwd=password, cursorclass=MySQLdb.cursors.DictCursor) if not conn: raise Exception('timeout') if dry_run: log.info('In dry_run mode, not changing anything') else: mysql_lib.enable_and_flush_activity_statistics(host_utils.HostAddr(hostname)) else: log.info("No recent user activity, skipping stats reset") # We still need to add it to the queue the first time. # Check if it was added recently and exit if it was if is_host_in_retirement_queue(hostname): return activity = RESET_STATS except MySQLdb.OperationalError as detail: (error_code, msg) = detail.args if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR: raise log.info('Could not connect to ' '{ip}'.format(ip=instance_metadata['internal_ip'])) activity = SHUTDOWN_MYSQL # We only want to add the host if it wasn't already in the queue if is_host_in_retirement_queue(hostname): return if dry_run: log.info('In dry_run mode, not changing anything') else: log_to_retirement_queue(hostname, instance_metadata['instance_id'], activity)