def check_for_user_activity(instance):
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
# check mysql activity
log.info('Checking activity on {instance}'.format(instance=instance['hostname']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=instance['internal_ip']))
activity = mysql_lib.get_user_activity(host_utils.HostAddr(instance['hostname']))
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected activity on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
log.info('Checking current connections on '
'{instance}'.format(instance=instance['hostname']))
connected_users = mysql_lib.get_connected_users(host_utils.HostAddr(instance['hostname']))
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
return False
def terminate_instances(hostname=None, dry_run=False):
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
terminate_instances = get_retirement_queue_servers(TERMINATE_INSTANCE)
botoconn = boto.ec2.connect_to_region('us-east-1')
if hostname:
if hostname in terminate_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
terminate_instances = {hostname: terminate_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for termination'.format(hostname=hostname))
return
for hostname in terminate_instances:
if hostname in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
log.warning("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
remove_from_retirement_queue(hostname)
continue
log.info('Confirming mysql is down on '
'{hostname}'.format(hostname=hostname))
try:
with timeout.timeout(3):
conn = MySQLdb.connect(
host=terminate_instances[hostname]['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
log.error('Did not get MYSQL_ERROR_CONN_HOST_ERROR, removing {} '
'from queue'.format(hostname))
conn.close()
remove_from_retirement_queue(hostname)
continue
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('MySQL is down')
log.info('Terminating instance '
'{instance}'.format(
instance=terminate_instances[hostname]['instance_id']))
if dry_run:
log.info('In dry_run mode, not changing state')
else:
botoconn.terminate_instances(
instance_ids=[terminate_instances[hostname]['instance_id']])
log_to_retirement_queue(
hostname, terminate_instances[hostname]['instance_id'],
TERMINATE_INSTANCE)
def terminate_instances(hostname=None, dry_run=False):
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
terminate_instances = get_retirement_queue_servers(TERMINATE_INSTANCE)
botoconn = boto.ec2.connect_to_region('us-east-1')
if hostname:
if hostname in terminate_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
terminate_instances = {hostname: terminate_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for termination'.format(hostname=hostname))
return
for hostname in terminate_instances:
if hostname in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
log.warning("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
remove_from_retirement_queue(hostname)
continue
log.info('Confirming mysql is down on '
'{hostname}'.format(hostname=hostname))
try:
with timeout.timeout(3):
pymysql.connect(host=terminate_instances[hostname]['internal_ip'],
user=username,
passwd=password,
cursorclass=pymysql.cursors.DictCursor)
log.error('Did not get MYSQL_ERROR_CONN_HOST_ERROR')
continue
except pymysql.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('MySQL is down')
log.info('Terminating instance '
'{instance}'.format(instance=terminate_instances[hostname]['instance_id']))
if dry_run:
log.info('In dry_run mode, not changing state')
else:
botoconn.terminate_instances(
instance_ids=[terminate_instances[hostname]['instance_id']])
log_to_retirement_queue(hostname,
terminate_instances[
hostname]['instance_id'],
TERMINATE_INSTANCE)
def add_to_queue(hostname, dry_run):
""" Add an instance to the retirement queue
Args:
hostname - The hostname of the instance to add to the retirement queue
"""
log.info('Adding server {hostname} to retirement '
'queue'.format(hostname=hostname))
if hostname in get_protected_hosts('set'):
raise Exception('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
# basic sanity check
zk = host_utils.MysqlZookeeper()
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
raise Exception("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
all_servers = environment_specific.get_all_server_metadata()
if not hostname in all_servers:
raise Exception('Host {hostname} is not cmdb'.format(hostname=hostname))
instance_metadata = all_servers[hostname]
log.info(instance_metadata)
username, password = mysql_lib.get_mysql_user_for_role('admin')
try:
log.info('Trying to reset user_statistics on ip '
'{ip}'.format(ip=instance_metadata['internal_ip']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance_metadata['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('timeout')
mysql_lib.enable_and_flush_activity_statistics(conn)
activity = RESET_STATS
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('Could not connect to '
'{ip}'.format(ip=instance_metadata['internal_ip']))
activity = SHUTDOWN_MYSQL
log_to_retirement_queue(hostname, instance_metadata['instance_id'],
activity)
def attack(attack_rsa_obj, publickey, cipher=[]):
"""Run fermat attack with a timeout
"""
try:
with timeout(seconds=attack_rsa_obj.args.timeout):
publickey.p, publickey.q = fermat(publickey.n)
except FactorizationError:
return (None, None)
if publickey.q is not None:
priv_key = PrivateKey(int(publickey.p), int(publickey.q),
int(publickey.e), int(publickey.n))
return (priv_key, None)
return (None, None)
def attack(attack_rsa_obj, publickey, cipher=[]):
"""Use primefac
"""
try:
with timeout(seconds=attack_rsa_obj.args.timeout):
result = list(
primefac(publickey.n, timeout=attack_rsa_obj.args.timeout))
except FactorizationError:
return (None, None)
if len(result) == 2:
publickey.p = int(result[0])
publickey.q = int(result[1])
priv_key = PrivateKey(int(publickey.p), int(publickey.q),
int(publickey.e), int(publickey.n))
return (priv_key, None)
return (None, None)
def attack(self, publickey, cipher=[]):
"""Run fermat attack with a timeout"""
try:
with timeout(seconds=self.timeout):
try:
publickey.p, publickey.q = self.fermat(publickey.n)
except TimeoutError:
return (None, None)
except FactorizationError:
return (None, None)
if publickey.p is not None and publickey.q is not None:
try:
priv_key = PrivateKey(int(publickey.p), int(publickey.q),
int(publickey.e))
return (priv_key, None)
except ValueError:
return (None, None)
return (None, None)
def take_flock_lock(file_name):
""" Take a flock for throw an exception
Args:
file_name - The name of the file to flock
Returns:
file_handle - This will be passed to release_flock_lock for relase
"""
success = False
try:
with timeout.timeout(1):
file_handle = open(file_name, 'w')
fcntl.flock(file_handle.fileno(), fcntl.LOCK_EX)
log.info('Lock taken')
success = True
except:
pass
# If success has not been set we will raise an exception just below
if not success:
raise Exception('Could not attain lock '
'on {file_name}'.format(file_name=file_name))
return file_handle
def take_flock_lock(file_name):
""" Take a flock for throw an exception
Args:
file_name - The name of the file to flock
Returns:
file_handle - This will be passed to release_flock_lock for relase
"""
success = False
try:
with timeout.timeout(1):
file_handle = open(file_name, 'w')
fcntl.flock(file_handle.fileno(), fcntl.LOCK_EX)
log.info('Lock taken')
success = True
except:
pass
# If success has not been set we will raise an exception just below
if not success:
raise Exception('Could not attain lock '
'on {file_name}'.format(file_name=file_name))
return file_handle
def check_for_user_activity(instance):
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
# check mysql activity
log.info('Checking activity on {instance}'.format(
instance=instance['hostname']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=instance['internal_ip']))
activity = mysql_lib.get_user_activity(
host_utils.HostAddr(instance['hostname']))
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected activity on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
log.info('Checking current connections on '
'{instance}'.format(instance=instance['hostname']))
connected_users = mysql_lib.get_connected_users(
host_utils.HostAddr(instance['hostname']))
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance['hostname'],
unexpected=','.join(unexpected)))
return True
return False
def add_to_queue(hostname, dry_run, skip_production_check=False):
""" Add an instance to the retirement queue
Args:
hostname - The hostname of the instance to add to the retirement queue
"""
log.info('Adding server {hostname} to retirement '
'queue'.format(hostname=hostname))
if hostname in get_protected_hosts('set'):
raise Exception('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
# basic sanity check
zk = host_utils.MysqlZookeeper()
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
if skip_production_check:
log.warning("It appears {instance} is in zk but "
"skip_production_check is set so continuing."
"".format(instance=instance))
else:
raise Exception("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
all_servers = environment_specific.get_all_server_metadata()
if hostname not in all_servers:
raise Exception(
'Host {hostname} is not cmdb'.format(hostname=hostname))
instance_metadata = all_servers[hostname]
log.info(instance_metadata)
username, password = mysql_lib.get_mysql_user_for_role('admin')
try:
if check_for_user_activity(instance_metadata):
log.info('Trying to reset user_statistics on ip '
'{ip}'.format(ip=instance_metadata['internal_ip']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance_metadata['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('timeout')
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
mysql_lib.enable_and_flush_activity_statistics(
host_utils.HostAddr(hostname))
else:
log.info("No recent user activity, skipping stats reset")
# We still need to add it to the queue the first time.
# Check if it was added recently and exit if it was
if is_host_in_retirement_queue(hostname):
return
activity = RESET_STATS
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('Could not connect to '
'{ip}'.format(ip=instance_metadata['internal_ip']))
activity = SHUTDOWN_MYSQL
# We only want to add the host if it wasn't already in the queue
if is_host_in_retirement_queue(hostname):
return
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
log_to_retirement_queue(hostname, instance_metadata['instance_id'],
activity)
def process_mysql_shutdown(hostname=None, dry_run=False):
""" Check stats, and shutdown MySQL instances"""
zk = host_utils.MysqlZookeeper()
username, password = mysql_lib.get_mysql_user_for_role('admin')
shutdown_instances = get_retirement_queue_servers(SHUTDOWN_MYSQL)
if hostname:
if hostname in shutdown_instances:
log.info('Only acting on {hostname}'.format(hostname=hostname))
shutdown_instances = {hostname: shutdown_instances[hostname]}
else:
log.info('Supplied host {hostname} is not ready '
'for shutdown'.format(hostname=hostname))
return
for instance in shutdown_instances:
if instance in get_protected_hosts('set'):
log.warning('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
remove_from_retirement_queue(hostname)
continue
for active_instance in zk.get_all_mysql_instances():
if active_instance.hostname == instance:
log.warning("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
remove_from_retirement_queue(instance)
continue
log.info('Checking activity on {instance}'.format(instance=instance))
# check mysql activity
with timeout.timeout(3):
conn = MySQLdb.connect(host=shutdown_instances[instance]['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('Could not connect to {ip}'
''.format(ip=shutdown_instances[instance]['internal_ip']))
activity = mysql_lib.get_user_activity(conn)
unexpected = set(activity.keys()).difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected acitivty on {instance} by user(s):'
'{unexpected}'.format(instance=instance,
unexpected=','.join(unexpected)))
continue
log.info('Checking current connections on '
'{instance}'.format(instance=instance))
connected_users = mysql_lib.get_connected_users(conn)
unexpected = connected_users.difference(IGNORABLE_USERS)
if unexpected:
log.error('Unexpected connection on {instance} by user(s):'
'{unexpected}'.format(instance=instance,
unexpected=','.join(unexpected)))
continue
# joining on a blank string as password must not have a space between
# the flag and the arg
if dry_run:
log.info('In dry_run mode, not changing state')
else:
log.info('Shuting down mysql on {instance}'.format(instance=instance))
mysql_lib.shutdown_mysql(host_utils.HostAddr(instance))
log_to_retirement_queue(instance,
shutdown_instances[instance]['instance_id'],
SHUTDOWN_MYSQL)
def add_to_queue(hostname, dry_run, skip_production_check=False):
""" Add an instance to the retirement queue
Args:
hostname - The hostname of the instance to add to the retirement queue
"""
log.info('Adding server {hostname} to retirement '
'queue'.format(hostname=hostname))
if hostname in get_protected_hosts('set'):
raise Exception('Host {hostname} is protected from '
'retirement'.format(hostname=hostname))
# basic sanity check
zk = host_utils.MysqlZookeeper()
for instance in zk.get_all_mysql_instances():
if instance.hostname == hostname:
if skip_production_check:
log.warning("It appears {instance} is in zk but "
"skip_production_check is set so continuing."
"".format(instance=instance))
else:
raise Exception("It appears {instance} is in zk. This is "
"very dangerous!".format(instance=instance))
all_servers = environment_specific.get_all_server_metadata()
if hostname not in all_servers:
raise Exception('Host {hostname} is not cmdb'.format(hostname=hostname))
instance_metadata = all_servers[hostname]
log.info(instance_metadata)
username, password = mysql_lib.get_mysql_user_for_role('admin')
try:
if check_for_user_activity(instance_metadata):
log.info('Trying to reset user_statistics on ip '
'{ip}'.format(ip=instance_metadata['internal_ip']))
with timeout.timeout(3):
conn = MySQLdb.connect(host=instance_metadata['internal_ip'],
user=username,
passwd=password,
cursorclass=MySQLdb.cursors.DictCursor)
if not conn:
raise Exception('timeout')
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
mysql_lib.enable_and_flush_activity_statistics(host_utils.HostAddr(hostname))
else:
log.info("No recent user activity, skipping stats reset")
# We still need to add it to the queue the first time.
# Check if it was added recently and exit if it was
if is_host_in_retirement_queue(hostname):
return
activity = RESET_STATS
except MySQLdb.OperationalError as detail:
(error_code, msg) = detail.args
if error_code != mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
raise
log.info('Could not connect to '
'{ip}'.format(ip=instance_metadata['internal_ip']))
activity = SHUTDOWN_MYSQL
# We only want to add the host if it wasn't already in the queue
if is_host_in_retirement_queue(hostname):
return
if dry_run:
log.info('In dry_run mode, not changing anything')
else:
log_to_retirement_queue(hostname, instance_metadata['instance_id'],
activity)