def checkup(self, command='sudo /etc/init.d/tomcat8 status'):
"""
Check takes a command, attempts
it and fails if it doesn't
have a 'Success' status in time.
Nicotine defaults to testing tomcat
because we have historically been a
java shop.
"""
self.command = command
try:
response = self.client.send_command(
InstanceIds=[self.instance_id],
DocumentName=self.command,
Parameters={'commands':[self.command]},)
# command needs some time
# to complete
# before checking command id
time.sleep(60)
self.command_id = response['Command']['CommandId']
vapor.vapors(f'command {self.command} with'
f'command id {self.command_id} for instance '
f'{self.instance_id} in env {self.env} '
'executed. response below:\n'
f'{response}')
return self.command_id
except Exception as e:
vapor.vapors('an exception was thrown during patch attempt'
f'for command: {self.command} with command id'
f'{self.command_id} for instance '
f'{self.instance_id} in env {self.env}. '
'exiting; response below:\n'
f'{response}', e)
raise
def smoke_free(self):
try:
jira_client = JIRA(self.jira_url, auth=(self.username, self.password))
response = jira_client.create_issue(project=self.project, summary=self.summary, description=self.description, issuetype=self.issuetype, customfield_SOMEINT=self.customfield_SOMEINT)
vapor.vapors('JIRA PBI created successfully! Congratualtions on being smoke free.')
except Exception as e:
vapor.vapors('something failed when creating JIRA ticket. see error below:\n', e)
raise
def anaphylaxis_check(self, name, reboot_command='sudo shutdown -r now', status_code: int):
self.name = name
self.reboot_command = reboot_command
self.status_code = status_code
# status success
# break
# status pending
# val == limit
# something
# status not success and not pending
# exit/possibly something
#
# ^ status(self, status, limit, attempts, action=[start, stop], force=[true, false])
# anaphylaxis algo
# attempt reboot
# status success
# system test
# status pending
# val == limit
# attempt force stop
# else
# continue
# status success
# system test
# status pending
# val == limit
# force stop
# attempt force stop
# status success
# attempt start
# status pending
# if val == limit
# sys.exit(-1)
# status not success and not pending
# sys.exit(-1)
# status not success and not pending
# sys.exit(-1)
# val != limit
# check again
# status not success and not pending
# sys.exit(-1)
# right now the only status_code
# is zero
if self.status_code == 0:
try:
response = client.send_command(
InstanceIds=[self.instance_id],
DocumentName="sudo_shutdown_now_r",
Parameters={'commands':[self.reboot_command]},)
vapor.vapors(f'reboot of {self.name} attempted. ' + \
'response below:\n{response}')
except as Exception e:
vapor.vapors('something went wrong when trying to ' + \
f'reboot {self.name}. exception below:\n', e)
sys.exit(-1)
def test_ami(self, ami_nm: str, ami_id: str):
"""
confirm epinephrine is
actually available. Exit in 4
hours or less if it is anything but.
"""
self.ami_nm = ami_nm
self.ami_id = ami_id
for idx, val in enumerate(range(122)):
self.ami_state = self.client.describe_images(
ImageIds=[self.ami_id])['Images'][0]['State']
if self.ami_state == 'available':
vapor.vapors(f'ami {self.ami_nm} with ami id {self.ami_id} '
'is available for epinephrine.')
break
elif self.ami_state == 'pending':
if val == 121:
vapor.vapors(
f'ami {self.ami_nm} with ami id {self.ami_id} '
'unable to form in 4 hours. If you think this ami'
'could just take that long then maybe investigate. '
'exiting.')
sys.exit(-1)
else:
vapor.vapors(
f'ami {self.ami_nm} with ami id {self.ami_id} still forming. sleeping 2 minutes.'
)
time.sleep(120)
continue
else:
vapor.vapors(f'ami state is {self.ami_state}. exiting')
sys.exit(-1)
def ami_name(self, nm: str, tm_stamp: str):
"""
create AMI with a unique name
"""
self.nm = nm
self.tm_stamp = tm_stamp
try:
self.ami_nm = f'nicotine_patch_{self.nm}_{self.tm_stamp}'
vapor.vapors(f'ami name: {self.ami_nm}')
return self.ami_nm
except Exception as e:
vapor.vapors(
'something went wrong '
f'when creating the ami name {self.ami_name}.\n', e)
raise
def get_tags(self):
""" get smoker tags """
try:
response = self.client.describe_tags(Filters=[
{
'Name': 'resource-id',
'Values': [self.instance_id],
},
], )
self.tags = response['Tags']
vapor.vapors(f"describe_tags response: {response}")
return self.tags
except Exception as e:
vapor.vapors('describe_tags error:\n', e)
raise
def create_ami_timestamp(self):
"""
create a unique timestamp
for our ami so that it is
stored as a unique object
in ec2 AMI
"""
try:
self.ami_timestamp = datetime.datetime.utcnow().strftime(
'%Y%m%d-%s')
vapor.vapors(f"ami_timestamp is: {self.ami_timestamp}")
return self.ami_timestamp
except Exception as e:
vapor.vapors('something went wrong with datetime:\n', e)
raise
def smoker():
"""
before we go see the
doctor we need to collect
some basic facts about
our smoker.
"""
parser = argparse.ArgumentParser()
try:
parser.add_argument('-e',
help='aws environment',
type=str,
action='store',
dest='env',
required=True)
parser.add_argument('-i',
help='instance id',
type=str,
action='store',
dest='instance_id',
required=True)
parser.add_argument(
'-s',
help=
'aws service with which to instantiate boto3 client, resource, etc',
type=str,
action='store',
dest='service',
required=False,
default='ec2')
parser.add_argument('--version',
action='version',
version='Nicotine 1.0')
args = parser.parse_args()
except Exception as e:
vapor.vapors('argparsing exception occurred:', e)
if args.env not in ['dev', 'qa', 'ct', 'pr']:
vapor.vapors('smoker must be in ' 'dev, qa, ct, or, pr. Exiting.')
sys.exit(-1)
return args
def anaphylaxis_check(self, name: str, status_code: int, reboot_command='sudo shutdown -r now'):
self.name = name
self.reboot_command = reboot_command
self.status_code = status_code
# right now the only status_code
# is zero
if self.status_code == 0:
try:
response = self.client.send_command(
InstanceIds=[self.instance_id],
DocumentName="sudo_shutdown_now_r",
Parameters={'commands':[self.reboot_command]},)
vapor.vapors(f'reboot of {self.name} attempted. '
'response below:\n{response}')
except as Exception e:
vapor.vapors('something went wrong when trying to '
f'reboot {self.name}. exception below:\n', e)
raise
def patch(self, patch_command='sudo yum update -y'):
"""
apply nicotine patch
via ssm and fail for
any exception (until exception
awareness can be improved)
"""
self.patch_command = patch_command
try:
response = client.send_command(
InstanceIds=[self.instance_id],
DocumentName="sudo_yum_update",
Parameters={'commands':[self.patch_command]},)
# command needs some time
# to complete
# before checking command id
time.sleep(60)
self.command_id = response['Command']['CommandId']
vapor.vapors(f'patch command {self.patch_command} with' + \
f'command id {self.command_id} for instance ' + \
f'{self.instance_id} in env {self.env} ' + \
'executed. response below:\n' + \
f'{response}')
vapor.vapors(f'\n{self.patch_command}')
except Exception as e:
vapor.vapors('an exception was thrown during patch attempt' + \
f'for command: {self.patch_command} with command id' + \
f'{self.command_id} for instance ' + \
f'{self.instance_id} in env {self.env}. ' + \
'exiting; response below:\n' + \
f'{response}', e)
def check_name(self, tg: list):
"""
We have to be certain
who this person is. Can't
just be administering epinephrine
to anyone now can we...
"""
self.tg = tg
try:
# does not handle
# two 'Names'
self.name = [
tag['Value'] for tag in self.tg if tag.get('Key') == 'Name'
]
self.smoker_name = self.name[0]
vapor.vapors(f'smoker name is: {self.smoker_name}')
return self.smoker_name
except IndexError as e:
vapor.vapors(
f'smoker with instance_id {self.instance_id} '
'has no name which is strange. exiting', e)
raise
def create_ami(self, nm: str):
"""
create an AMI in case
epinephrine needs to be
aministered
"""
self.nm = nm
try:
response = self.client.create_image(Description=self.nm,
Name=self.nm,
InstanceId=self.instance_id,
NoReboot=True)
self.ami_id = response['ImageId']
vapor.vapors(f'ami {self.nm} has ami id '
f'{self.ami_id}. ami creation attempt '
f'is below:\n'
f'{response}')
return self.ami_id
except Exception as e:
vapor.vapors(
'something went wrong '
f'when creating ami {self.nm}:\n', e)
raise
def allergy_test(self, cmd_id):
"""
test if the patch has completed
in 10 minutes or less. if so: reboot, else:
fail and rollback via GoodIntentions class.
"""
try:
for idx, val in enumerate(range(11)):
response = self.client.get_command_invocation(
InstanceId=self.instance_id,
CommandId=self.cmd_id)
if response['Status'] == 'Success':
vapor.vapors(f'patch command {self.patch_command} with command id '
f'{self.cmd_id} for instance '
f'{self.instance_id} in env {self.env} '
'succeeded. response below:\n'
f'{repsonse}')
return 0
elif response['Status'] in ['Delayed', 'InProgress', 'Pending']:
if val == 10:
vapor.vapors(f'{self.patch_command} with command id'
f'{self.cmd_id} for instance '
f'{self.instance_id} in env {self.env} '
f'has taken 10 minutes to attempt '
f'command execution. rolling back. '
'failed response below:\n'
f'{response}')
# launch epinephrine box from prep epinephrine ami
# if elb is relevant, roll that machine in behind elb
# stop or force stop jacked box
# if elb is relevant, roll jacked box out from behind elb
# system test new box
sys.exit(-1)
else:
vapor.vapors(f'patch command {self.patch_command} with command id '
f'{self.cmd_id} for instance '
f'{self.instance_id} in env {self.env} has a status of '
f"{response['Status']}. sleeping for 60 seconds "
'and then checking status again.')
time.sleep(60)
continue
class Checkup:
"""
Checkup runs system tests.
Inherit Checkup to implement
system tests for any stack
you like.
"""
def __init__(self, env: str, instance_id: str,
service: str, client: Doctor):
self.env = env
self.instance_id = instance_id
self.service = service
self.client = client
def checkup(self, command='sudo /etc/init.d/tomcat8 status'):
"""
Check takes a command, attempts
it and fails if it doesn't
have a 'Success' status in time.
Nicotine defaults to testing tomcat
because we have historically been a
java shop.
"""
self.command = command
try:
response = self.client.send_command(
InstanceIds=[self.instance_id],
DocumentName=self.command,
Parameters={'commands':[self.command]},)
# command needs some time
# to complete
# before checking command id
time.sleep(60)
self.command_id = response['Command']['CommandId']
vapor.vapors(f'command {self.command} with'
f'command id {self.command_id} for instance '
f'{self.instance_id} in env {self.env} '
'executed. response below:\n'
f'{response}')
return self.command_id
except Exception as e:
vapor.vapors('an exception was thrown during patch attempt'
f'for command: {self.command} with command id'
f'{self.command_id} for instance '
f'{self.instance_id} in env {self.env}. '
'exiting; response below:\n'
f'{response}', e)
raise
def checkup_satisfaction(self, self.cmd_id):
"""
test if the checkup command has completed
in 2 minutes or less. if so: run next
command or exit.
"""
try:
for idx, val in enumerate(range(3)):
response = self.client.get_command_invocation(
InstanceId=self.instance_id,
CommandId=self.cmd_id)
if response['Status'] == 'Success':
vapor.vapors(f'command {self.command} with command id '
f'{self.cmd_id} for instance '
f'{self.instance_id} in env {self.env} '
'succeeded. smoke_free!!! (creating jira ticket)')
return 0
elif response['Status'] in ['Delayed', 'InProgress', 'Pending']:
if val == 2:
vapor.vapors(f'{self.command} with command id'
f'{self.cmd_id} for instance '
f'{self.instance_id} in env {self.env} '
f'has taken 2 minutes to attempt '
f'command execution. rolling back. '
'failed response below:\n'
f'{response}')
# launch epinephrine box from prep epinephrine ami
# if elb is relevant, roll that machine in behind elb
# stop or force stop jacked box
# if elb is relevant, roll jacked box out from behind elb
# system test new box
sys.exit(-1)
else:
vapor.vapors(f'command {self.command} with command id '
f'{self.cmd_id} for instance '
f'{self.instance_id} in env {self.env} has a status of '
f'{response['Status']}. sleeping for 60 seconds '
'and then checking status again.')
time.sleep(60)
class NicotinePatch:
"""
NicotinePatch is a glorified
'sudo yum update -y' and
'sudo shutdown -r now', but it
executes these commands with intelligence
and automated instance rollback
"""
def __init__(self, env, instance_id, service):
self.env = env
self.instance_id = instance_id
self.service = service
doc = Doctor(self, self.env, self.instance_id, self.service)
client = doc.create_client()
instance = doc.create_instance()
def patch(self, patch_command='sudo yum update -y'):
"""
apply nicotine patch
via ssm and fail for
any exception (until exception
awareness can be improved)
"""
self.patch_command = patch_command
try:
response = client.send_command(
InstanceIds=[self.instance_id],
DocumentName="sudo_yum_update",
Parameters={'commands':[self.patch_command]},)
# command needs some time
# to complete
# before checking command id
time.sleep(60)
self.command_id = response['Command']['CommandId']
vapor.vapors(f'patch command {self.patch_command} with' + \
f'command id {self.command_id} for instance ' + \
f'{self.instance_id} in env {self.env} ' + \
'executed. response below:\n' + \
f'{response}')
vapor.vapors(f'\n{self.patch_command}')
except Exception as e:
vapor.vapors('an exception was thrown during patch attempt' + \
f'for command: {self.patch_command} with command id' + \
f'{self.command_id} for instance ' + \
f'{self.instance_id} in env {self.env}. ' + \
'exiting; response below:\n' + \
f'{response}', e)
def allergy_test(self, self.cmd_id):
"""
test if the patch has completed
in 10 minutes or less. if so: reboot, else:
fail and rollback via GoodIntentions class.
"""
try:
for idx, val in enumerate(range(11)):
response = client.get_command_invocation(
InstanceId=self.instance_id,
CommandId=self.cmd_id)
if response['Status'] == 'Success':
vapor.vapors(f'patch command {self.patch_command} with command id ' + \
f'{self.cmd_id} for instance ' + \
f'{self.instance_id} in env {self.env} ' + \
'succeeded. response below:\n'
f'{repsonse})
return 0
elif response['Status'] in ['Delayed', 'InProgress', 'Pending']:
if val == 10:
vapor.vapors(f'{self.patch_command} with command id' + \
f'{self.cmd_id} for instance ' + \
f'{self.instance_id} in env {self.env} ' + \
f'has taken 10 minutes to attempt ' + \
f'command execution. rolling back. ' + \
'failed response below:\n'
f'{response}')
# launch epinephrine box from prep epinephrine ami
# if elb is relevant, roll that machine in behind elb
# stop or force stop jacked box
# if elb is relevant, roll jacked box out from behind elb
# system test new box
sys.exit(-1)
else:
vapor.vapors(f'patch command {self.patch_command} with command id ' + \
f'{self.cmd_id} for instance ' + \
f'{self.instance_id} in env {self.env} has a status of ' + \
f'{response['Status']}. sleeping for 60 seconds ' + \
'and then checking status again.')
time.sleep(60)
if self.status_code == 0:
try:
response = self.client.send_command(
InstanceIds=[self.instance_id],
DocumentName="sudo_shutdown_now_r",
Parameters={'commands':[self.reboot_command]},)
vapor.vapors(f'reboot of {self.name} attempted. '
'response below:\n{response}')
except as Exception e:
vapor.vapors('something went wrong when trying to '
f'reboot {self.name}. exception below:\n', e)
raise
else:
vapor.vapors('anaphylaxis_check method received a status_code '
'of {self.status_code} and cannot continue. response is '
'below. exiting.')
sys.exit(-1)
# nice case for recursion
for idx,val in enumerate(range(11)):
# {'Code': 16, 'Name': 'running'}
if self.instance.state == 16:
vapor.vapors(f'reboot of {self.name} successful. system '
'testing will now proceed.')
break
# {'Code': 0, 'Name': 'pending'}
elif self.instance.state == 0:
if val == 10:
vapor.vapors(f'reboot of {self.name} has taken '
'10 minutes. exiting.')
def main():
# smoker is not
# a class and does not
# have vapors
#
# passing
try:
smoke = smoker()
vapor.vapors('smoker instantiated successfully')
except Exception as e:
vapor.vapors('smoker instantiation failed', e)
raise
try:
sec = secret()
vapor.vapors('secrets collected successfully')
except Exception as e:
vapor.vapors('failed to collect secrets', e)
raise
# try/excepts are built-in
# unlike with smoker
#
# passing
clear = Clearance('/Users/me/.aws/credentials', '/usr/local/bin/fed')
clear.check_botocore()
clear.check_boto3()
clear.check_credentials()
clear.check_fed()
clear.check_fed_version()
# try/except
# not built-in to
# Doctor class
#
# passing
try:
doc = Doctor(smoke.env, smoke.instance_id)
vapor.vapors('doctor instantiation successful')
except Exception as e:
vapor.vapors('doctor instantiation failed', e)
raise
try:
client = doc.create_client()
vapor.vapors(f'boto3 {doc.service} client instantiation successful')
except Exception as e:
vapor.vapors(f'boto3 {doc.service} client instantiation failed', e)
raise
try:
res = doc.create_resource()
vapor.vapors(f'boto3 {doc.service} resource instantiation successful')
except Exception as e:
vapor.vapors(f'boto3 {doc.service} resource instantiation failed', e)
raise
try:
instance = doc.create_instance()
vapor.vapors(f'boto3 {doc.service} instance instantiation successful')
except Exception as e:
vapor.vapors(f'boto3 {doc.service} instance instantiation failed', e)
raise
try:
prep = PrepEpinephrine(smoke.env, smoke.instance_id, client)
vapor.vapors('epinephrine successfully prepped')
except Exception as e:
vapor.vapors('epinephrine prep failed', e)
raise
# built-in try/except
timestamp = prep.create_ami_timestamp()
tags = prep.get_tags()
name = prep.check_name(tags)
ami_name = prep.ami_name(name, timestamp)
ami_id = prep.create_ami(ami_name)
prep.test_ami(ami_name, ami_id)
try:
nic = NicotinePatch(smoke.env, smoke.instance_id, client, instance)
vapor.vapors('nicotine patch instantiated ' 'successfully')
except Exception as e:
vapor.vapors('nicotine patch instnatiation ' 'failed.', e)
raise
if self.status_code == 0:
try:
response = client.send_command(
InstanceIds=[self.instance_id],
DocumentName="sudo_shutdown_now_r",
Parameters={'commands':[self.reboot_command]},)
vapor.vapors(f'reboot of {self.name} attempted. ' + \
'response below:\n{response}')
except as Exception e:
vapor.vapors('something went wrong when trying to ' + \
f'reboot {self.name}. exception below:\n', e)
sys.exit(-1)
else:
vapor.vapors('anaphylaxis_check method received a status_code ' + \
'of {self.status_code} and cannot continue. response is ' + \
'below. exiting.')
sys.exit(-1)
# nice case for recursion
for idx,val in enumerate(range(11)):
# {'Code': 16, 'Name': 'running'}
if instance.state == 16:
vapor.vapors(f'reboot of {self.name} successful. system ' + \
'testing will now proceed.')
break
# {'Code': 0, 'Name': 'pending'}
elif instance.state == 0:
if val == 10:
vapor.vapors(f'reboot of {self.name} has taken ' + \
