def _query_stack(options): if options['profile'] != '': conn = cloudformation.connect_to_region(options['region'], profile_name=options['profile']) else: conn = cloudformation.connect_to_region(options['region'], aws_access_key_id=options['AWS_ACCESS_KEY_ID'], aws_secret_access_key=options['AWS_SECRET_ACCESS_KEY']) stack_status_filters=[ 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE' ] next_token=None end_list=False json_all_stacks = dict() json_all_stacks['stacks'] = dict() ## Using no regex will improve speed of the process if options['regex'] == False: for stack_name in options['stacks']: debug("Will discover %s" % (stack_name)) discover_stack(options, conn, json_all_stacks, stack_name) else: while (end_list == False): stacks = conn.list_stacks(stack_status_filters=stack_status_filters, next_token=next_token) next_token = stacks.next_token if next_token == None: end_list=True for stack in stacks: debug("%s : %s" % (stack.stack_name, stack.stack_status)) for match_stack in options['stacks']: a = re.compile(match_stack) result = a.match(stack.stack_name) if result == None: continue debug("Will discover %s" % (stack.stack_name)) discover_stack(options, conn, json_all_stacks, stack.stack_name) if len(json_all_stacks['stacks']) > 0: print json.dumps(json_all_stacks, indent=4, sort_keys=True)
def new_stack(options): consul_conn = Consul(options.host, options.port) cf_conn = cloudformation.connect_to_region(options.region) ec2_conn = ec2.connect_to_region(options.region) stack_validator = NestedStackValidator(cf_conn) stack_service = StackService(cf_conn, ec2_conn, consul_conn, stack_validator) return Stack(stack_service, options)
def __init__(self, environment, deployment, region, zone, template=template): # Create connections to AWS components self.cfn_connection = cfn.connect_to_region(region) self.sns_connection = sns.connect_to_region(region) self.vpc_connection = vpc.connect_to_region(region) self.iam_connection = iam.connect_to_region("universal") # Temporary python class -> directory name hack self.lab_dir = self.__class__.__name__.lower() self.stack_name = "-".join( [self.lab_dir, environment, deployment, region, zone]) if environment != '': self.notification_arns = self.get_sns_topic( "cloudformation-notifications-" + environment) self.parameters = [] # Prepare the CFN template self.template_url = "/".join([ os.path.dirname(os.path.realpath(__file__)), self.lab_dir, vpc_provider, template ]) self.template_body = self.read_file(self.template_url, max_template_size) self.validate_template()
def cleanup_stacks(): cfn = cloudformation.connect_to_region("eu-west-1") cfn.delete_stack("cfn-sphere-test-instances") wait_for_stack_to_disappear("cfn-sphere-test-instances") cfn.delete_stack("cfn-sphere-test-vpc") wait_for_stack_to_disappear("cfn-sphere-test-vpc")
def get_cfn_connection(self): """ We persist the CFN connection so that we don't create a new session with each request """ if not self.cfn_connection: self.cfn_connection = cloudformation.connect_to_region(self.config.get('boto').get('region_name')) return self.cfn_connection
def get_cf_stack(self, stack, resources=False): """ Get information on parameters, outputs and resources from a stack and cache it """ if not resources: if not self.cf_stacks.has_key(stack): #We don't have this stack in the cache already so we need to pull it from CF cfconn = cloudformation.connect_to_region(self.region) self.cf_stacks[stack] = cfconn.describe_stacks(stack)[0] return self.cf_stacks[stack] else: if not self.cf_stacks_resources.has_key(stack): cfconn = cloudformation.connect_to_region(self.region) the_stack = self.get_cf_stack(stack=stack, resources=False) self.cf_stacks_resources[stack] = the_stack.list_resources() return self.cf_stacks_resources[stack]
def get_cf_stack(self, stack, resources = False): """ Get information on parameters, outputs and resources from a stack and cache it """ if not resources: if not self.cf_stacks.has_key(stack): #We don't have this stack in the cache already so we need to pull it from CF cfconn = cloudformation.connect_to_region(self.region) self.cf_stacks[stack] = cfconn.describe_stacks(stack)[0] return self.cf_stacks[stack] else: if not self.cf_stacks_resources.has_key(stack): cfconn = cloudformation.connect_to_region(self.region) the_stack = self.get_cf_stack(stack = stack, resources = False) self.cf_stacks_resources[stack] = the_stack.list_resources() return self.cf_stacks_resources[stack]
def setUpClass(cls): test_resources_dir = get_resources_dir() cls.cfn_conn = cloudformation.connect_to_region("eu-west-1") cls.config = Config(config_file=os.path.join(test_resources_dir, "stacks.yml")) cls.stack_handler = StackActionHandler(cls.config) LOGGER.info("Syncing stacks") cls.stack_handler.create_or_update_stacks()
def upload(region, cfbucket, credentials, stack_name, json_stack_def): template_key = upload_s3_unique(region, cfbucket, credentials, json_stack_def) template_url = template_key.generate_url(expires_in=0, query_auth=False) logger.debug('Template available in s3 at url: %s', template_url) logger.debug('Uploading stack definition with name: %s', stack_name) cf = cloudformation.connect_to_region(region, **credentials) cf.create_stack(stack_name, capabilities=['CAPABILITY_IAM'], template_url=template_url) logger.debug('Done uploading stack definition')
def destroy_stacks(mmw_config, aws_profile, **kwargs): """Destroy stacks that are associated with stack_color""" region = mmw_config['Region'] stack_color = kwargs['stack_color'] cfn_conn = cfn.connect_to_region(region, profile_name=aws_profile) color_tag = ('StackColor', stack_color.capitalize()) [stack.delete() for stack in cfn_conn.describe_stacks() if color_tag in stack.tags.items()]
def delete(self, resource): if resource.wrapped.stack_status in Stack.VALID_TARGET_STATES: self.logger.info("Skipping deletion: State '{0}' is a valid target state.".format( resource.wrapped.stack_status)) return if self.dry_run: return self.logger.info("Initiating deletion sequence for {stack_name}.".format(**vars(resource.wrapped))) connection = cloudformation.connect_to_region(resource.region) connection.delete_stack(resource.wrapped.stack_id)
def _get_cf_connection(): conn = cloudformation.connect_to_region(os.environ['AWS_DEFAULT_REGION']) try: conn.region return conn except AttributeError: print "Unable to obtain a Cloud Formation connection. Review the " \ "README to ensure you've provided the correct environment to " \ "connect to AWS." exit()
def get_connection(): try: connection = connect_to_region( aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region_name=AWS_CLOUDFORMATION_REGION, is_secure=True, ) except BotoServerError, e: raise Exception("Unable to connect to CloudFormation: %s" % e.error_message)
def __init__(self, region="eu-west-1"): logging.getLogger('boto').setLevel(logging.FATAL) self.logger = get_logger() self.conn = cloudformation.connect_to_region(region) if not self.conn: self.logger.error("Could not connect to cloudformation API in {0}. Invalid region?".format(region)) raise Exception("Got None connection object") self.logger.debug("Connected to cloudformation API at {0} with access key id: {1}".format( region, self.conn.aws_access_key_id))
def __init__(self, profile): config = AwsConfigMFA() region = "us-east-1" r = config.get("profile %s" % profile, "region") if r != None: region = r creds = config.getTokenCredentials(profile) self.cf = cloudformation.connect_to_region(region, aws_access_key_id=creds['access_key'], aws_secret_access_key=creds['secret_key'], security_token = creds['session_token'])
def wait_for_stack_to_disappear(stack_name): LOGGER.info("Waiting for stack {0} to disappear".format(stack_name)) cfn = cloudformation.connect_to_region("eu-west-1") for i in range(1, 30): stacks = [stack.stack_name for stack in cfn.list_stacks() if stack.stack_status != "DELETE_COMPLETE"] if stack_name not in stacks: return time.sleep(10) i += 1 raise Exception("Timeout occured waiting for stack {0} to disappear".format(stack_name))
def get_instance_ids_for_stack(stackname): """ For a given cloudformation stack, return all of the instance ids, eg ["i-f1430877", "i-g3430877", ...] """ instance_ids_for_stack = [] region = cloudformation.connect_to_region(DEFAULT_REGION) stack_resources = region.describe_stack_resources(stackname) for stack_resource in stack_resources: if stack_resource.resource_type == "AWS::EC2::Instance": instance_ids_for_stack.append(stack_resource.physical_resource_id) return instance_ids_for_stack
def destroy_stacks(icp_config, aws_profile, **kwargs): """Destroy stacks that are associated with stack_color""" region = icp_config['Region'] stack_type = icp_config['StackType'] stack_color = kwargs['stack_color'] cfn_conn = cfn.connect_to_region(region, profile_name=aws_profile) stack_tag = ('StackType', stack_type) color_tag = ('StackColor', stack_color.capitalize()) [stack.delete() for stack in cfn_conn.describe_stacks() if color_tag in stack.tags.items() and stack_tag in stack.tags.items()]
def fetch_unwanted_resources(self): for region in self.regions: connection = cloudformation.connect_to_region(region.name) unwanted_states = set(connection.valid_states) unwanted_states.remove("DELETE_COMPLETE") resources = connection.list_stacks(stack_status_filters=list(unwanted_states)) or [] for resource in resources: resource_wrapper = Resource(resource, region.name) if resource.stack_name in self.ignored_resources: self.logger.info('IGNORE ' + self.to_string(resource_wrapper)) continue yield resource_wrapper
def delete(self, resource): if resource.wrapped.stack_status in Stack.VALID_TARGET_STATES: warnings.warn( Warning( "Skipping deletion: State '{0}' is a valid target state.". format(resource.wrapped.stack_status))) if self.dry_run: return self.logger.info( "Initiating deletion sequence for {stack_name}.".format( **vars(resource.wrapped))) connection = cloudformation.connect_to_region(resource.region) connection.delete_stack(resource.wrapped.stack_id)
def __init__(self, profile): config = AwsConfigMFA() region = "us-east-1" r = config.get("profile %s" % profile, "region") if r != None: region = r creds = config.getTokenCredentials(profile) self.cf = cloudformation.connect_to_region( region, aws_access_key_id=creds["access_key"], aws_secret_access_key=creds["secret_key"], security_token=creds["session_token"], )
def populate_params(self, current_cf_stacks): #If we have no parameters in the yaml file, set params to an empty dict and return true if self.yaml_params is None: self.params = {} return True if self.deps_met(current_cf_stacks): cfconn = cloudformation.connect_to_region(self.region) for param in self.yaml_params.keys(): if type(self.yaml_params[param]) is dict: #Static value set, so use it if self.yaml_params[param].has_key('value'): self.params[param] = str(self.yaml_params[param]['value']) #No static value set, but if we have a source, type and variable can try getting from CF elif self.yaml_params[param].has_key('source') and self.yaml_params[param].has_key('type') and self.yaml_params[param].has_key('variable'): if self.yaml_params[param]['source'] == self.mega_stack_name: source_stack = self.yaml_params[param]['source'] elif self.yaml_params[param]['source'][:1] == '-': source_stack = self.yaml_params[param]['source'][1:] else: source_stack = "%s-%s" % (self.mega_stack_name, self.yaml_params[param]['source']) self.params[param] = self.get_value_from_cf( source_stack = source_stack, var_type = self.yaml_params[param]['type'], var_name = self.yaml_params[param]['variable'] ) #If self.yaml_params[param] is a list it means there is an array of vars we need to turn into a comma sep list. elif type(self.yaml_params[param]) is list: param_list = [] for item in self.yaml_params[param]: if type(item) is dict: #Static value set, so use it if item.has_key('value'): param_list.append(str(item['value'])) #No static value set, but if we have a source, type and variable can try getting from CF elif item.has_key('source') and item.has_key('type') and item.has_key('variable'): if item['source'] == self.mega_stack_name: source_stack = item['source'] else: source_stack = "%s-%s" % (self.mega_stack_name, item['source']) param_list.append(self.get_value_from_cf( source_stack = source_stack, var_type = item['type'], var_name = item['variable'] )) else: print "Error in yaml file, %s in parameter list for %s stack. Can't populate." % (self.yaml_params[param],self.name) exit(1) self.params[param] = ','.join(param_list) return True else: return False
def __init__(self, region="eu-west-1"): logging.getLogger("boto").setLevel(logging.FATAL) self.logger = get_logger() self.conn = cloudformation.connect_to_region(region) if not self.conn: self.logger.error( "Could not connect to cloudformation API in {0}. Invalid region?" .format(region)) raise Exception("Got None connection object") self.logger.debug( "Connected to cloudformation API at {0} with access key id: {1}". format(region, self.conn.aws_access_key_id))
def _query_stack(options): if options["profile"] != "": conn = cloudformation.connect_to_region(options["region"], profile_name=options["profile"]) else: conn = cloudformation.connect_to_region( options["region"], aws_access_key_id=options["AWS_ACCESS_KEY_ID"], aws_secret_access_key=options["AWS_SECRET_ACCESS_KEY"], ) stack_status_filters = ["CREATE_IN_PROGRESS", "CREATE_COMPLETE", "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE"] next_token = None end_list = False json_all_stacks = dict() json_all_stacks["stacks"] = dict() ## Using no regex will improve speed of the process if options["regex"] == False: for stack_name in options["stacks"]: debug("Will discover %s" % (stack_name)) discover_stack(options, conn, json_all_stacks, stack_name) else: while end_list == False: stacks = conn.list_stacks(stack_status_filters=stack_status_filters, next_token=next_token) next_token = stacks.next_token if next_token == None: end_list = True for stack in stacks: debug("%s : %s" % (stack.stack_name, stack.stack_status)) for match_stack in options["stacks"]: a = re.compile(match_stack) result = a.match(stack.stack_name) if result == None: continue debug("Will discover %s" % (stack.stack_name)) discover_stack(options, conn, json_all_stacks, stack.stack_name) if len(json_all_stacks["stacks"]) > 0: print json.dumps(json_all_stacks, indent=4, sort_keys=True)
def populate_params(self, current_cf_stacks): #If we have no parameters in the yaml file, set params to an empty dict and return true if self.yaml_params is None: self.params = {} return True if self.deps_met(current_cf_stacks): cfconn = cloudformation.connect_to_region(self.region) for param in self.yaml_params.keys(): if type(self.yaml_params[param]) is dict: #Static value set, so use it if self.yaml_params[param].has_key('value'): self.params[param] = str(self.yaml_params[param]['value']) #No static value set, but if we have a source, type and variable can try getting from CF elif self.yaml_params[param].has_key('source') and self.yaml_params[param].has_key('type') and self.yaml_params[param].has_key('variable'): if self.yaml_params[param]['source'] == self.mega_stack_name: source_stack = self.yaml_params[param]['source'] else: source_stack = "%s-%s" % (self.mega_stack_name, self.yaml_params[param]['source']) self.params[param] = self.get_value_from_cf( source_stack = source_stack, var_type = self.yaml_params[param]['type'], var_name = self.yaml_params[param]['variable'] ) #If self.yaml_params[param] is a list it means there is an array of vars we need to turn into a comma sep list. elif type(self.yaml_params[param]) is list: param_list = [] for item in self.yaml_params[param]: if type(item) is dict: #Static value set, so use it if item.has_key('value'): param_list.append(str(item['value'])) #No static value set, but if we have a source, type and variable can try getting from CF elif item.has_key('source') and item.has_key('type') and item.has_key('variable'): if item['source'] == self.mega_stack_name: source_stack = item['source'] else: source_stack = "%s-%s" % (self.mega_stack_name, item['source']) param_list.append(self.get_value_from_cf( source_stack = source_stack, var_type = item['type'], var_name = item['variable'] )) else: print "Error in yaml file, %s in parameter list for %s stack. Can't populate." % (self.yaml_params[param],self.name) exit(1) self.params[param] = ','.join(param_list) return True else: return False
def __init__(self, aws_access_key_id, aws_secret_access_key, security_token): self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key self.security_token = security_token self.cf = cloudformation.connect_to_region( region_name='us-west-2', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, security_token=security_token) self.ec2 = ec2.connect_to_region( 'us-west-2', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, security_token=security_token)
def connect_cloudformation(): """ Connect to AWS CloudFormation :returns: boto.cloudformation.connection """ try: return cloudformation.connect_to_region( config.get_environment_option('region'), aws_access_key_id=config.get_environment_option('access-key-id'), aws_secret_access_key=config.get_environment_option( 'secret-access-key')) except Exception as err: logger.error( 'A problem occurred connecting to AWS CloudFormation: {}'.format( err)) raise
def connect_cloudformation(): """ Connect to AWS CloudFormation :returns: boto.cloudformation.connection """ try: return cloudformation.connect_to_region( config.get_environment_option('region'), aws_access_key_id=config.get_environment_option( 'access-key-id'), aws_secret_access_key=config.get_environment_option( 'secret-access-key')) except Exception as err: logger.error( 'A problem occurred connecting to AWS CloudFormation: {}'.format( err)) raise
def __init__(self, environment, deployment, region, zone, aws_access_key_id, aws_secret_access_key): # Create connections to AWS components self.cfn_connection = cfn.connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) self.sns_connection = sns.connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) self.vpc_connection = vpc.connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) # Temporary python class -> directory name hack lab_dir = self.__class__.__name__.lower() self.stack_name = "-".join([lab_dir, environment, deployment, region, zone]) self.notification_arns = self.get_sns_topic("cloudformation-notifications-" + environment) self.parameters = [] # Prepare the CFN template self.template_url = "/".join([os.path.dirname(os.path.realpath(__file__)), lab_dir, vpc_provider, template]) self.template_body = self.read_file(self.template_url, max_template_size) self.validate_template()
def get_aws_connection(service, region_name='eu-west-1'): credentials = { 'aws_access_key_id': config.AWS_ACCESS_KEY_ID, 'aws_secret_access_key': config.AWS_SECERET_ACCESS_KEY } if service == 'ec2': return ec2.connect_to_region(region_name, **credentials) elif service == 'cloudformation': return cloudformation.connect_to_region(region_name, **credentials) elif service == 'autoscale': return autoscale.connect_to_region(region_name, **credentials) elif service == 'elb': return elb.connect_to_region(region_name, **credentials) elif service == 'cloudwatch': return cloudwatch.connect_to_region(region_name, **credentials) else: raise Exception("Unkown service '%s'" % service)
def fetch_unwanted_resources(self): for region_name in self.region_names: connection = cloudformation.connect_to_region(region_name) unwanted_states = set(connection.valid_states) unwanted_states.remove("DELETE_COMPLETE") resources = connection.list_stacks( stack_status_filters=list(unwanted_states)) or [] for resource in resources: resource_wrapper = Resource( resource=resource, resource_type=self.resource_type, resource_id=resource.stack_id, creation_date=resource.creation_time, region=region_name) if resource.stack_name in self.ignored_resources: self.logger.info('IGNORE ' + self.to_string(resource_wrapper)) continue yield resource_wrapper
def __init__(self, module, env, cluster, zone, template_name='template.json', **optionals): # debug and/or dry-run mode self.debug = optionals['debug'] self.dry_run = optionals['dry_run'] self.enable_debug() self.env = env.strip().lower() self.cluster = snake_case_with_dashes(cluster.strip()) self.zone = zone self.template_name = template_name # for stack names & resources names in template, use only non-default names self.non_default_cluster = self.cluster if self.cluster not in [default_cluster_name] else None # stack name self.module_name = snake_case_with_dashes(module.strip()) self.stack_name = "-".join(filter(None, [self.module_name, self.env, self.non_default_cluster, self.zone])) # params self.parameters = [] # stack tags self.stack_tags = { '{0}:minion:env'.format(org_name): self.env, '{0}:minion:cluster'.format(org_name): self.cluster, '{0}:minion:module'.format(org_name): self.module_name } # Create connections to AWS components aws_access_key_id = get_env_variable('MINION_ACCESS_KEY_ID') aws_secret_access_key = get_env_variable('MINION_SECRET_ACCESS_KEY') self.cfn_connection = cfn.connect_to_region(aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) self.vpc_connection = vpc.connect_to_region(aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) self.iam_connection = iam.connect_to_region("universal", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
def get_cfn_conn(): # type: () -> CloudFormationConnection """Creates CloudFormation connection object using region data in playbooks/vars/main.yaml""" # Open vars/main.yaml. Used to get AWS region with open('playbooks/vars/main.yaml', mode='r') as f: try: vars = yaml.load(f) except yaml.YAMLError as e: print( "There was a problem loading 'playbooks/vars/main.yaml. \n%s" % e.message) exit(1) # Get CloudFormation connection try: cfn_conn = connect_to_region(region_name=vars['region']) except Exception as e: print("There was a problem connecting to AWS. \n%s" % e.message) exit(1) return cfn_conn
def wait_until_stack_create_complete(stackname): """ Wait until this stack is complete. In other words, when there is a stack event with: event.resource_type = AWS::CloudFormation::Stack event.resource_status = CREATE_COMPLETE """ for x in xrange(NUM_RETRIES): print ("Waiting for {} to finish launching. Attempt: {}".format(stackname, x)) region = cloudformation.connect_to_region(DEFAULT_REGION) try: region.describe_stacks(stackname) except BotoServerError as bse: print ("Exception describing stack: {}, exception: {}. Retrying.".format(stackname, bse)) continue stack_events = region.describe_stack_events(stackname) for stack_event in stack_events: if stack_event.stack_name != stackname: print ( "Ignoring {} since it's stack name is {} instead of {}".format( stack_event, stack_event.stack_name, stackname ) ) continue if ( stack_event.resource_type == "AWS::CloudFormation::Stack" and stack_event.resource_status == "CREATE_COMPLETE" ): print ("Stack {} has successfully been created".format(stackname)) return # didn't find it, lets wait and try again time.sleep(5)
def _launch_cfn(self): """ Sets up stack and launches it. """ self.set_up_stack() self.boto_conn = cloudformation.connect_to_region(region_name=self.aws_region, profile_name=self.aws_profile) parameters = [] for param, input_name in self.input_wiring.iteritems(): try: parameters.append((param, self.get_input(input_name))) except MKInputError: pass # check to see if stack exists try: self.stack = self.boto_conn.describe_stacks(self.stack_name)[0] except BotoServerError: # it would be great if we could more granularly check the error self.boto_conn.create_stack(self.stack_name, tags=self.get_raw_tags(), template_body=self.to_json(), parameters=parameters) self.logger.info('Stack %s created', self.stack_name)
def get_value_from_cf(self, source_stack, var_type, var_name): """ Get a variable from a existing cloudformation stack, var_type should be parameter, resource or output. If using resource, provide the logical ID and this will return the Physical ID """ cfconn = cloudformation.connect_to_region(self.region) the_stack = self.get_cf_stack(stack = source_stack) if var_type == 'parameter': for p in the_stack.parameters: if str(p.key) == var_name: return str(p.value) elif var_type == 'output': for o in the_stack.outputs: if str(o.key) == var_name: return str(o.value) elif var_type == 'resource': for r in self.get_cf_stack(stack = source_stack, resources = True): if str(r.logical_resource_id) == var_name: return str(r.physical_resource_id) else: print "Error: invalid var_type passed to get_value_from_cf, needs to be parameter, resource or output. Not: %s" % (var_type) exit(1)
def get_value_from_cf(self, source_stack, var_type, var_name): """ Get a variable from a existing cloudformation stack, var_type should be parameter, resource or output. If using resource, provide the logical ID and this will return the Physical ID """ cfconn = cloudformation.connect_to_region(self.region) the_stack = self.get_cf_stack(stack = source_stack) if var_type == 'parameter': for p in the_stack.parameters: if str(p.key) == var_name: return str(p.value) elif var_type == 'output': for o in the_stack.outputs: if str(o.key) == var_name: return str(o.key) elif var_type == 'resource': for r in self.get_cf_stack(stack = source_stack, resources = True): if str(r.logical_resource_id) == var_name: return str(r.physical_resource_id) else: print "Error: invalid var_type passed to get_value_from_cf, needs to be parameter, resource or output. Not: %s" % (var_type) exit(1)
def wait_until_stack_create_complete(stackname): """ Wait until this stack is complete. In other words, when there is a stack event with: event.resource_type = AWS::CloudFormation::Stack event.resource_status = CREATE_COMPLETE """ for x in xrange(NUM_RETRIES): print("Waiting for {} to finish launching. Attempt: {}".format( stackname, x)) region = cloudformation.connect_to_region(DEFAULT_REGION) try: region.describe_stacks(stackname) except BotoServerError as bse: print("Exception describing stack: {}, exception: {}. Retrying.". format(stackname, bse)) continue stack_events = region.describe_stack_events(stackname) for stack_event in stack_events: if stack_event.stack_name != stackname: print("Ignoring {} since it's stack name is {} instead of {}". format(stack_event, stack_event.stack_name, stackname)) continue if stack_event.resource_type == "AWS::CloudFormation::Stack": if stack_event.resource_status in [ "CREATE_COMPLETE", "UPDATE_COMPLETE" ]: print("Stack {} has successfully been created/updated". format(stackname)) return # didn't find it, lets wait and try again time.sleep(5)
def _get_connection(profile_name, region_name): LOGGER.info( 'Connecting to AWS using Profile: {profile} in Region {region}'.format( profile=profile_name, region=region_name)) return cloudformation.connect_to_region(region_name=region_name, profile_name=profile_name)
def __init__(self, yamlFile): self.logger = logging.getLogger(__name__) #load the yaml file and turn it into a dict thefile = open(yamlFile, 'r') self.stackDict = yaml.safe_load(thefile) #Make sure there is only one top level element in the yaml file if len(self.stackDict.keys()) != 1: self.logger.critical( "Need one and only one mega stack name at the top level, found %s" % len(self.stackDict.keys())) exit(1) #How we know we only have one top element, that must be the mega stack name self.name = self.stackDict.keys()[0] #Find and set the mega stacks region. Exit if we can't find it if self.stackDict[self.name].has_key('region'): self.region = self.stackDict[self.name]['region'] else: self.logger.critical( "No region specified for mega stack, don't know where to build it." ) exit(1) self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', []) if isinstance(self.sns_topic_arn, str): self.sns_topic_arn = [self.sns_topic_arn] for topic in self.sns_topic_arn: if topic.split(':')[3] != self.region: self.logger.critical("SNS Topic %s is not in the %s region." % (topic, self.region)) exit(1) #Array for holding CFStack objects once we create them self.stack_objs = [] #Get the names of the sub stacks from the yaml file and sort in array self.cf_stacks = self.stackDict[self.name]['stacks'].keys() #Megastack holds the connection to cloudformation and list of stacks currently in our region #Stops us making lots of calls to cloudformation API for each stack try: self.cfconn = cloudformation.connect_to_region(self.region) self.cf_desc_stacks = self.cfconn.describe_stacks() except boto.exception.NoAuthHandlerFound as e: self.logger.critical( "No credentials found for connecting to cloudformation: %s" % e) exit(1) #iterate through the stacks in the yaml file and create CFstack objects for them for stack_name in self.cf_stacks: the_stack = self.stackDict[self.name]['stacks'][stack_name] if type(the_stack) is dict: local_sns_arn = the_stack.get('sns-topic-arn', self.sns_topic_arn) if isinstance(local_sns_arn, str): local_sns_arn = [local_sns_arn] for topic in local_sns_arn: if topic.split(':')[3] != self.region: self.logger.critical( "SNS Topic %s is not in the %s region." % (topic, self.region)) exit(1) if the_stack.has_key('cf_template'): self.stack_objs.append( CFStack(mega_stack_name=self.name, name=stack_name, params=the_stack['params'], template_name=the_stack['cf_template'], region=self.region, sns_topic_arn=local_sns_arn, depends_on=the_stack['depends']))
elif opt in ("-y"): options['noconfirmation'] = True elif opt in ("-s", "--stack"): try: re.compile(arg) except Exception, e: logger.error("Invalid regular expression %s", arg) sys.exit(1) options['stacks'].append(arg) if not options['region']: logger.error('Needs region specified.') sys.exit(1) if options['profile']: conn = cloudformation.connect_to_region(options['region'], profile_name=options['profile']) elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ: conn = cloudformation.connect_to_region( options['region'], aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'] ) else: logger.error('Missing AWS credentials. Check your environment/profile.') sys.exit(1) if options['list_stacks']: for s in get_safe_stack_list(conn, options['stacks']): print s.stack_name sys.exit()
if __name__ == '__main__': args = docopt(__doc__, version='Lambda Chat AWS Resources 0.2') config = load_config() print_cf_template = args['cf'] or args['launch'] try: if print_cf_template: template = generate_cf_template() print(template.to_json()) if (args['cf']): sys.exit(1) # Get a connection to AWS CloudFormation in the given region conn = cloudformation.connect_to_region(args['--region']) if (args['launch']): launch(args, config, conn, template) elif (args['update']): update(args, config, conn, template) elif (args['delete']): delete(args, config, conn) elif (args['output']): output(args, config, conn) except Exception, e:
def __init__(self, yamlFile): self.logger = logging.getLogger(__name__) #load the yaml file and turn it into a dict thefile = open(yamlFile, 'r') self.stackDict = yaml.safe_load(thefile) #Make sure there is only one top level element in the yaml file if len(self.stackDict.keys()) != 1: self.logger.critical("Need one and only one mega stack name at the top level, found %s" % len(self.stackDict.keys())) exit(1) #How we know we only have one top element, that must be the mega stack name self.name = self.stackDict.keys()[0] #Find and set the mega stacks region. Exit if we can't find it if self.stackDict[self.name].has_key('region'): self.region = self.stackDict[self.name]['region'] else: self.logger.critical("No region specified for mega stack, don't know where to build it.") exit(1) self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', []) if isinstance(self.sns_topic_arn, str): self.sns_topic_arn = [self.sns_topic_arn] for topic in self.sns_topic_arn: if topic.split(':')[3] != self.region: self.logger.critical("SNS Topic %s is not in the %s region." % (topic, self.region)) exit(1) #Array for holding CFStack objects once we create them self.stack_objs = [] #Get the names of the sub stacks from the yaml file and sort in array self.cf_stacks = self.stackDict[self.name]['stacks'].keys() #Megastack holds the connection to cloudformation and list of stacks currently in our region #Stops us making lots of calls to cloudformation API for each stack try: self.cfconn = cloudformation.connect_to_region(self.region) self.cf_desc_stacks = self.cfconn.describe_stacks() except boto.exception.NoAuthHandlerFound as e: self.logger.critical("No credentials found for connecting to cloudformation: %s" % e ) exit(1) #iterate through the stacks in the yaml file and create CFstack objects for them for stack_name in self.cf_stacks: the_stack = self.stackDict[self.name]['stacks'][stack_name] if type(the_stack) is dict: local_sns_arn = the_stack.get('sns-topic-arn', self.sns_topic_arn) if isinstance(local_sns_arn, str): local_sns_arn = [local_sns_arn] for topic in local_sns_arn: if topic.split(':')[3] != self.region: self.logger.critical("SNS Topic %s is not in the %s region." % (topic, self.region)) exit(1) if the_stack.has_key('cf_template'): self.stack_objs.append( CFStack( mega_stack_name=self.name, name=stack_name, params=the_stack['params'], template_name=the_stack['cf_template'], region=self.region, sns_topic_arn=local_sns_arn, depends_on=the_stack['depends'] ) )
def cloudformation(self): if not hasattr(self, '_cloudformation'): self._cloudformation = cloudformation.connect_to_region( self.region) return self._cloudformation
print ("Stack creation failed.") exit() # Deleting the older stack def delete_stack(connection_cf,stack_name): stack_list=connection_cf.list_stacks(stack_status_filters=['CREATE_COMPLETE']) for stack in stack_list: if (stack.stack_name).startswith('T3Hydration'): if stack.stack_name != stack_name: try: delete_stack(connection_cf,stack.stack_name) except Exception as e: print "Error occurred while deleting the stack." print e # Creating a cloud formation stack ends here. # Main functions starts here. connection_ec2 = ec2.connect_to_region(aws_region,aws_access_key_id=AWS_ACCESS_KEY_ID,aws_secret_access_key=AWS_SECRET_ACCESS_KEY) connection_cf = cf.connect_to_region(aws_region,aws_access_key_id=AWS_ACCESS_KEY_ID,aws_secret_access_key=AWS_SECRET_ACCESS_KEY) timestr = time.strftime("%Y%m%d") stack_name="T3Hydration-"+str(timestr) create_snapshots(connection_ec2) get_latest_ami_id(connection_ec2) update_cf_template() with open(file_name,'r') as f: tags = {'Name':stack_name} create_stack(connection_cf,stack_name,f.read(),tags,[]) #delete_stack(connection_cf,stack_name)
def __init__(self, yamlFile): self.logger = logging.getLogger(__name__) # load the yaml file and turn it into a dict thefile = open(yamlFile, 'r') rendered_file = pystache.render(thefile.read(), dict(os.environ)) self.stackDict = yaml.safe_load(rendered_file) # Make sure there is only one top level element in the yaml file if len(self.stackDict.keys()) != 1: error_message = ("Need one and only one mega stack name at the" + " top level, found %s") self.logger.critical(error_message % len(self.stackDict.keys())) exit(1) # Now we know we only have one top element, # that must be the mega stack name self.name = self.stackDict.keys()[0] # Find and set the mega stacks region. Exit if we can't find it if 'region' in self.stackDict[self.name]: self.region = self.stackDict[self.name]['region'] else: self.logger.critical("No region specified for mega stack," + " don't know where to build it.") exit(1) if 'account_id' in self.stackDict[self.name]: # Get the account ID for the current AWS credentials iamconn = iam.connect_to_region(self.region) user_response = iamconn.get_user()['get_user_response'] user_result = user_response['get_user_result'] account_id = user_result['user']['arn'].split(':')[4] # Check if the current account ID matches the stack's account ID if account_id != str(self.stackDict[self.name]['account_id']): self.logger.critical("Account ID of stack does not match the" + " account ID of your AWS credentials.") exit(1) self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', []) if isinstance(self.sns_topic_arn, str): self.sns_topic_arn = [self.sns_topic_arn] for topic in self.sns_topic_arn: if topic.split(':')[3] != self.region: self.logger.critical("SNS Topic %s is not in the %s region." % (topic, self.region)) exit(1) self.global_tags = self.stackDict[self.name].get('tags', {}) # Array for holding CFStack objects once we create them self.stack_objs = [] # Get the names of the sub stacks from the yaml file and sort in array self.cf_stacks = self.stackDict[self.name]['stacks'].keys() # Megastack holds the connection to CloudFormation and list of stacks # currently in our region stops us making lots of calls to # CloudFormation API for each stack try: self.cfconn = cloudformation.connect_to_region(self.region) self.cf_desc_stacks = self._describe_all_stacks() except boto.exception.NoAuthHandlerFound as exception: self.logger.critical( "No credentials found for connecting to CloudFormation: %s" % exception) exit(1) # iterate through the stacks in the yaml file and create CFstack # objects for them for stack_name in self.cf_stacks: the_stack = self.stackDict[self.name]['stacks'][stack_name] if type(the_stack) is dict: if the_stack.get('disable', False): warn_message = ("Stack %s is disabled by configuration" + " directive. Skipping") self.logger.warning(warn_message % stack_name) continue local_sns_arn = the_stack.get('sns-topic-arn', self.sns_topic_arn) if isinstance(local_sns_arn, str): local_sns_arn = [local_sns_arn] for topic in local_sns_arn: if topic.split(':')[3] != self.region: error_message = "SNS Topic %s is not in the %s region." self.logger.critical(error_message % (topic, self.region)) exit(1) local_tags = the_stack.get('tags', {}) merged_tags = dict(self.global_tags.items() + local_tags.items()) # Add static cumulus-stack tag merged_tags['cumulus-stack'] = self.name if 'cf_template' in the_stack: self.stack_objs.append( CFStack( mega_stack_name=self.name, name=stack_name, params=the_stack.get('params'), template_name=the_stack['cf_template'], region=self.region, sns_topic_arn=local_sns_arn, depends_on=the_stack.get('depends'), tags=merged_tags ) )
def __init__(self, region, profile): self.region = region self.cf = cf.connect_to_region(self.region, profile_name=profile)
if __name__ == '__main__': args = docopt(__doc__, version='Lambda Chat AWS Resources 0.2') config = load_config() print_cf_template = args['cf'] or args['launch'] try: if print_cf_template: template = generate_cf_template() print(template.to_json()) if (args['cf']): sys.exit(1) # Get a connection to AWS CloudFormation in the given region conn = cloudformation.connect_to_region( args['--region'], profile_name=args['--profile']) if (args['launch']): launch(args, config, conn, template) elif (args['update']): update(args, config, conn, template) elif (args['delete']): delete(args, config, conn) elif (args['output']): output(args, config, conn) except Exception, e:
def __init__(self, aws_access_key_id, aws_secret_access_key, security_token): self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key self.security_token = security_token self.cf = cloudformation.connect_to_region(region_name = 'us-west-2', aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, security_token = security_token) self.ec2 = ec2.connect_to_region('us-west-2', aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, security_token = security_token)
def __init__(self): self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) self.test_resources_dir = self._get_resources_dir() self.cfn_conn = cloudformation.connect_to_region("eu-west-1") self.config = Config(config_file=os.path.join(self.test_resources_dir, "stacks.yml"))