def checkup(self, command='sudo /etc/init.d/tomcat8 status'): """ Check takes a command, attempts it and fails if it doesn't have a 'Success' status in time. Nicotine defaults to testing tomcat because we have historically been a java shop. """ self.command = command try: response = self.client.send_command( InstanceIds=[self.instance_id], DocumentName=self.command, Parameters={'commands':[self.command]},) # command needs some time # to complete # before checking command id time.sleep(60) self.command_id = response['Command']['CommandId'] vapor.vapors(f'command {self.command} with' f'command id {self.command_id} for instance ' f'{self.instance_id} in env {self.env} ' 'executed. response below:\n' f'{response}') return self.command_id except Exception as e: vapor.vapors('an exception was thrown during patch attempt' f'for command: {self.command} with command id' f'{self.command_id} for instance ' f'{self.instance_id} in env {self.env}. ' 'exiting; response below:\n' f'{response}', e) raise
def smoke_free(self): try: jira_client = JIRA(self.jira_url, auth=(self.username, self.password)) response = jira_client.create_issue(project=self.project, summary=self.summary, description=self.description, issuetype=self.issuetype, customfield_SOMEINT=self.customfield_SOMEINT) vapor.vapors('JIRA PBI created successfully! Congratualtions on being smoke free.') except Exception as e: vapor.vapors('something failed when creating JIRA ticket. see error below:\n', e) raise
def anaphylaxis_check(self, name, reboot_command='sudo shutdown -r now', status_code: int): self.name = name self.reboot_command = reboot_command self.status_code = status_code # status success # break # status pending # val == limit # something # status not success and not pending # exit/possibly something # # ^ status(self, status, limit, attempts, action=[start, stop], force=[true, false]) # anaphylaxis algo # attempt reboot # status success # system test # status pending # val == limit # attempt force stop # else # continue # status success # system test # status pending # val == limit # force stop # attempt force stop # status success # attempt start # status pending # if val == limit # sys.exit(-1) # status not success and not pending # sys.exit(-1) # status not success and not pending # sys.exit(-1) # val != limit # check again # status not success and not pending # sys.exit(-1) # right now the only status_code # is zero if self.status_code == 0: try: response = client.send_command( InstanceIds=[self.instance_id], DocumentName="sudo_shutdown_now_r", Parameters={'commands':[self.reboot_command]},) vapor.vapors(f'reboot of {self.name} attempted. ' + \ 'response below:\n{response}') except as Exception e: vapor.vapors('something went wrong when trying to ' + \ f'reboot {self.name}. exception below:\n', e) sys.exit(-1)
def test_ami(self, ami_nm: str, ami_id: str): """ confirm epinephrine is actually available. Exit in 4 hours or less if it is anything but. """ self.ami_nm = ami_nm self.ami_id = ami_id for idx, val in enumerate(range(122)): self.ami_state = self.client.describe_images( ImageIds=[self.ami_id])['Images'][0]['State'] if self.ami_state == 'available': vapor.vapors(f'ami {self.ami_nm} with ami id {self.ami_id} ' 'is available for epinephrine.') break elif self.ami_state == 'pending': if val == 121: vapor.vapors( f'ami {self.ami_nm} with ami id {self.ami_id} ' 'unable to form in 4 hours. If you think this ami' 'could just take that long then maybe investigate. ' 'exiting.') sys.exit(-1) else: vapor.vapors( f'ami {self.ami_nm} with ami id {self.ami_id} still forming. sleeping 2 minutes.' ) time.sleep(120) continue else: vapor.vapors(f'ami state is {self.ami_state}. exiting') sys.exit(-1)
def ami_name(self, nm: str, tm_stamp: str): """ create AMI with a unique name """ self.nm = nm self.tm_stamp = tm_stamp try: self.ami_nm = f'nicotine_patch_{self.nm}_{self.tm_stamp}' vapor.vapors(f'ami name: {self.ami_nm}') return self.ami_nm except Exception as e: vapor.vapors( 'something went wrong ' f'when creating the ami name {self.ami_name}.\n', e) raise
def get_tags(self): """ get smoker tags """ try: response = self.client.describe_tags(Filters=[ { 'Name': 'resource-id', 'Values': [self.instance_id], }, ], ) self.tags = response['Tags'] vapor.vapors(f"describe_tags response: {response}") return self.tags except Exception as e: vapor.vapors('describe_tags error:\n', e) raise
def create_ami_timestamp(self): """ create a unique timestamp for our ami so that it is stored as a unique object in ec2 AMI """ try: self.ami_timestamp = datetime.datetime.utcnow().strftime( '%Y%m%d-%s') vapor.vapors(f"ami_timestamp is: {self.ami_timestamp}") return self.ami_timestamp except Exception as e: vapor.vapors('something went wrong with datetime:\n', e) raise
def smoker(): """ before we go see the doctor we need to collect some basic facts about our smoker. """ parser = argparse.ArgumentParser() try: parser.add_argument('-e', help='aws environment', type=str, action='store', dest='env', required=True) parser.add_argument('-i', help='instance id', type=str, action='store', dest='instance_id', required=True) parser.add_argument( '-s', help= 'aws service with which to instantiate boto3 client, resource, etc', type=str, action='store', dest='service', required=False, default='ec2') parser.add_argument('--version', action='version', version='Nicotine 1.0') args = parser.parse_args() except Exception as e: vapor.vapors('argparsing exception occurred:', e) if args.env not in ['dev', 'qa', 'ct', 'pr']: vapor.vapors('smoker must be in ' 'dev, qa, ct, or, pr. Exiting.') sys.exit(-1) return args
def anaphylaxis_check(self, name: str, status_code: int, reboot_command='sudo shutdown -r now'): self.name = name self.reboot_command = reboot_command self.status_code = status_code # right now the only status_code # is zero if self.status_code == 0: try: response = self.client.send_command( InstanceIds=[self.instance_id], DocumentName="sudo_shutdown_now_r", Parameters={'commands':[self.reboot_command]},) vapor.vapors(f'reboot of {self.name} attempted. ' 'response below:\n{response}') except as Exception e: vapor.vapors('something went wrong when trying to ' f'reboot {self.name}. exception below:\n', e) raise
def patch(self, patch_command='sudo yum update -y'): """ apply nicotine patch via ssm and fail for any exception (until exception awareness can be improved) """ self.patch_command = patch_command try: response = client.send_command( InstanceIds=[self.instance_id], DocumentName="sudo_yum_update", Parameters={'commands':[self.patch_command]},) # command needs some time # to complete # before checking command id time.sleep(60) self.command_id = response['Command']['CommandId'] vapor.vapors(f'patch command {self.patch_command} with' + \ f'command id {self.command_id} for instance ' + \ f'{self.instance_id} in env {self.env} ' + \ 'executed. response below:\n' + \ f'{response}') vapor.vapors(f'\n{self.patch_command}') except Exception as e: vapor.vapors('an exception was thrown during patch attempt' + \ f'for command: {self.patch_command} with command id' + \ f'{self.command_id} for instance ' + \ f'{self.instance_id} in env {self.env}. ' + \ 'exiting; response below:\n' + \ f'{response}', e)
def check_name(self, tg: list): """ We have to be certain who this person is. Can't just be administering epinephrine to anyone now can we... """ self.tg = tg try: # does not handle # two 'Names' self.name = [ tag['Value'] for tag in self.tg if tag.get('Key') == 'Name' ] self.smoker_name = self.name[0] vapor.vapors(f'smoker name is: {self.smoker_name}') return self.smoker_name except IndexError as e: vapor.vapors( f'smoker with instance_id {self.instance_id} ' 'has no name which is strange. exiting', e) raise
def create_ami(self, nm: str): """ create an AMI in case epinephrine needs to be aministered """ self.nm = nm try: response = self.client.create_image(Description=self.nm, Name=self.nm, InstanceId=self.instance_id, NoReboot=True) self.ami_id = response['ImageId'] vapor.vapors(f'ami {self.nm} has ami id ' f'{self.ami_id}. ami creation attempt ' f'is below:\n' f'{response}') return self.ami_id except Exception as e: vapor.vapors( 'something went wrong ' f'when creating ami {self.nm}:\n', e) raise
def allergy_test(self, cmd_id): """ test if the patch has completed in 10 minutes or less. if so: reboot, else: fail and rollback via GoodIntentions class. """ try: for idx, val in enumerate(range(11)): response = self.client.get_command_invocation( InstanceId=self.instance_id, CommandId=self.cmd_id) if response['Status'] == 'Success': vapor.vapors(f'patch command {self.patch_command} with command id ' f'{self.cmd_id} for instance ' f'{self.instance_id} in env {self.env} ' 'succeeded. response below:\n' f'{repsonse}') return 0 elif response['Status'] in ['Delayed', 'InProgress', 'Pending']: if val == 10: vapor.vapors(f'{self.patch_command} with command id' f'{self.cmd_id} for instance ' f'{self.instance_id} in env {self.env} ' f'has taken 10 minutes to attempt ' f'command execution. rolling back. ' 'failed response below:\n' f'{response}') # launch epinephrine box from prep epinephrine ami # if elb is relevant, roll that machine in behind elb # stop or force stop jacked box # if elb is relevant, roll jacked box out from behind elb # system test new box sys.exit(-1) else: vapor.vapors(f'patch command {self.patch_command} with command id ' f'{self.cmd_id} for instance ' f'{self.instance_id} in env {self.env} has a status of ' f"{response['Status']}. sleeping for 60 seconds " 'and then checking status again.') time.sleep(60) continue
class Checkup: """ Checkup runs system tests. Inherit Checkup to implement system tests for any stack you like. """ def __init__(self, env: str, instance_id: str, service: str, client: Doctor): self.env = env self.instance_id = instance_id self.service = service self.client = client def checkup(self, command='sudo /etc/init.d/tomcat8 status'): """ Check takes a command, attempts it and fails if it doesn't have a 'Success' status in time. Nicotine defaults to testing tomcat because we have historically been a java shop. """ self.command = command try: response = self.client.send_command( InstanceIds=[self.instance_id], DocumentName=self.command, Parameters={'commands':[self.command]},) # command needs some time # to complete # before checking command id time.sleep(60) self.command_id = response['Command']['CommandId'] vapor.vapors(f'command {self.command} with' f'command id {self.command_id} for instance ' f'{self.instance_id} in env {self.env} ' 'executed. response below:\n' f'{response}') return self.command_id except Exception as e: vapor.vapors('an exception was thrown during patch attempt' f'for command: {self.command} with command id' f'{self.command_id} for instance ' f'{self.instance_id} in env {self.env}. ' 'exiting; response below:\n' f'{response}', e) raise def checkup_satisfaction(self, self.cmd_id): """ test if the checkup command has completed in 2 minutes or less. if so: run next command or exit. """ try: for idx, val in enumerate(range(3)): response = self.client.get_command_invocation( InstanceId=self.instance_id, CommandId=self.cmd_id) if response['Status'] == 'Success': vapor.vapors(f'command {self.command} with command id ' f'{self.cmd_id} for instance ' f'{self.instance_id} in env {self.env} ' 'succeeded. smoke_free!!! (creating jira ticket)') return 0 elif response['Status'] in ['Delayed', 'InProgress', 'Pending']: if val == 2: vapor.vapors(f'{self.command} with command id' f'{self.cmd_id} for instance ' f'{self.instance_id} in env {self.env} ' f'has taken 2 minutes to attempt ' f'command execution. rolling back. ' 'failed response below:\n' f'{response}') # launch epinephrine box from prep epinephrine ami # if elb is relevant, roll that machine in behind elb # stop or force stop jacked box # if elb is relevant, roll jacked box out from behind elb # system test new box sys.exit(-1) else: vapor.vapors(f'command {self.command} with command id ' f'{self.cmd_id} for instance ' f'{self.instance_id} in env {self.env} has a status of ' f'{response['Status']}. sleeping for 60 seconds ' 'and then checking status again.') time.sleep(60)
class NicotinePatch: """ NicotinePatch is a glorified 'sudo yum update -y' and 'sudo shutdown -r now', but it executes these commands with intelligence and automated instance rollback """ def __init__(self, env, instance_id, service): self.env = env self.instance_id = instance_id self.service = service doc = Doctor(self, self.env, self.instance_id, self.service) client = doc.create_client() instance = doc.create_instance() def patch(self, patch_command='sudo yum update -y'): """ apply nicotine patch via ssm and fail for any exception (until exception awareness can be improved) """ self.patch_command = patch_command try: response = client.send_command( InstanceIds=[self.instance_id], DocumentName="sudo_yum_update", Parameters={'commands':[self.patch_command]},) # command needs some time # to complete # before checking command id time.sleep(60) self.command_id = response['Command']['CommandId'] vapor.vapors(f'patch command {self.patch_command} with' + \ f'command id {self.command_id} for instance ' + \ f'{self.instance_id} in env {self.env} ' + \ 'executed. response below:\n' + \ f'{response}') vapor.vapors(f'\n{self.patch_command}') except Exception as e: vapor.vapors('an exception was thrown during patch attempt' + \ f'for command: {self.patch_command} with command id' + \ f'{self.command_id} for instance ' + \ f'{self.instance_id} in env {self.env}. ' + \ 'exiting; response below:\n' + \ f'{response}', e) def allergy_test(self, self.cmd_id): """ test if the patch has completed in 10 minutes or less. if so: reboot, else: fail and rollback via GoodIntentions class. """ try: for idx, val in enumerate(range(11)): response = client.get_command_invocation( InstanceId=self.instance_id, CommandId=self.cmd_id) if response['Status'] == 'Success': vapor.vapors(f'patch command {self.patch_command} with command id ' + \ f'{self.cmd_id} for instance ' + \ f'{self.instance_id} in env {self.env} ' + \ 'succeeded. response below:\n' f'{repsonse}) return 0 elif response['Status'] in ['Delayed', 'InProgress', 'Pending']: if val == 10: vapor.vapors(f'{self.patch_command} with command id' + \ f'{self.cmd_id} for instance ' + \ f'{self.instance_id} in env {self.env} ' + \ f'has taken 10 minutes to attempt ' + \ f'command execution. rolling back. ' + \ 'failed response below:\n' f'{response}') # launch epinephrine box from prep epinephrine ami # if elb is relevant, roll that machine in behind elb # stop or force stop jacked box # if elb is relevant, roll jacked box out from behind elb # system test new box sys.exit(-1) else: vapor.vapors(f'patch command {self.patch_command} with command id ' + \ f'{self.cmd_id} for instance ' + \ f'{self.instance_id} in env {self.env} has a status of ' + \ f'{response['Status']}. sleeping for 60 seconds ' + \ 'and then checking status again.') time.sleep(60)
if self.status_code == 0: try: response = self.client.send_command( InstanceIds=[self.instance_id], DocumentName="sudo_shutdown_now_r", Parameters={'commands':[self.reboot_command]},) vapor.vapors(f'reboot of {self.name} attempted. ' 'response below:\n{response}') except as Exception e: vapor.vapors('something went wrong when trying to ' f'reboot {self.name}. exception below:\n', e) raise else: vapor.vapors('anaphylaxis_check method received a status_code ' 'of {self.status_code} and cannot continue. response is ' 'below. exiting.') sys.exit(-1) # nice case for recursion for idx,val in enumerate(range(11)): # {'Code': 16, 'Name': 'running'} if self.instance.state == 16: vapor.vapors(f'reboot of {self.name} successful. system ' 'testing will now proceed.') break # {'Code': 0, 'Name': 'pending'} elif self.instance.state == 0: if val == 10: vapor.vapors(f'reboot of {self.name} has taken ' '10 minutes. exiting.')
def main(): # smoker is not # a class and does not # have vapors # # passing try: smoke = smoker() vapor.vapors('smoker instantiated successfully') except Exception as e: vapor.vapors('smoker instantiation failed', e) raise try: sec = secret() vapor.vapors('secrets collected successfully') except Exception as e: vapor.vapors('failed to collect secrets', e) raise # try/excepts are built-in # unlike with smoker # # passing clear = Clearance('/Users/me/.aws/credentials', '/usr/local/bin/fed') clear.check_botocore() clear.check_boto3() clear.check_credentials() clear.check_fed() clear.check_fed_version() # try/except # not built-in to # Doctor class # # passing try: doc = Doctor(smoke.env, smoke.instance_id) vapor.vapors('doctor instantiation successful') except Exception as e: vapor.vapors('doctor instantiation failed', e) raise try: client = doc.create_client() vapor.vapors(f'boto3 {doc.service} client instantiation successful') except Exception as e: vapor.vapors(f'boto3 {doc.service} client instantiation failed', e) raise try: res = doc.create_resource() vapor.vapors(f'boto3 {doc.service} resource instantiation successful') except Exception as e: vapor.vapors(f'boto3 {doc.service} resource instantiation failed', e) raise try: instance = doc.create_instance() vapor.vapors(f'boto3 {doc.service} instance instantiation successful') except Exception as e: vapor.vapors(f'boto3 {doc.service} instance instantiation failed', e) raise try: prep = PrepEpinephrine(smoke.env, smoke.instance_id, client) vapor.vapors('epinephrine successfully prepped') except Exception as e: vapor.vapors('epinephrine prep failed', e) raise # built-in try/except timestamp = prep.create_ami_timestamp() tags = prep.get_tags() name = prep.check_name(tags) ami_name = prep.ami_name(name, timestamp) ami_id = prep.create_ami(ami_name) prep.test_ami(ami_name, ami_id) try: nic = NicotinePatch(smoke.env, smoke.instance_id, client, instance) vapor.vapors('nicotine patch instantiated ' 'successfully') except Exception as e: vapor.vapors('nicotine patch instnatiation ' 'failed.', e) raise
if self.status_code == 0: try: response = client.send_command( InstanceIds=[self.instance_id], DocumentName="sudo_shutdown_now_r", Parameters={'commands':[self.reboot_command]},) vapor.vapors(f'reboot of {self.name} attempted. ' + \ 'response below:\n{response}') except as Exception e: vapor.vapors('something went wrong when trying to ' + \ f'reboot {self.name}. exception below:\n', e) sys.exit(-1) else: vapor.vapors('anaphylaxis_check method received a status_code ' + \ 'of {self.status_code} and cannot continue. response is ' + \ 'below. exiting.') sys.exit(-1) # nice case for recursion for idx,val in enumerate(range(11)): # {'Code': 16, 'Name': 'running'} if instance.state == 16: vapor.vapors(f'reboot of {self.name} successful. system ' + \ 'testing will now proceed.') break # {'Code': 0, 'Name': 'pending'} elif instance.state == 0: if val == 10: vapor.vapors(f'reboot of {self.name} has taken ' + \