def _query_stack(options):
if options['profile'] != '':
conn = cloudformation.connect_to_region(options['region'], profile_name=options['profile'])
else:
conn = cloudformation.connect_to_region(options['region'], aws_access_key_id=options['AWS_ACCESS_KEY_ID'], aws_secret_access_key=options['AWS_SECRET_ACCESS_KEY'])
stack_status_filters=[ 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE' ]
next_token=None
end_list=False
json_all_stacks = dict()
json_all_stacks['stacks'] = dict()
## Using no regex will improve speed of the process
if options['regex'] == False:
for stack_name in options['stacks']:
debug("Will discover %s" % (stack_name))
discover_stack(options, conn, json_all_stacks, stack_name)
else:
while (end_list == False):
stacks = conn.list_stacks(stack_status_filters=stack_status_filters, next_token=next_token)
next_token = stacks.next_token
if next_token == None:
end_list=True
for stack in stacks:
debug("%s : %s" % (stack.stack_name, stack.stack_status))
for match_stack in options['stacks']:
a = re.compile(match_stack)
result = a.match(stack.stack_name)
if result == None:
continue
debug("Will discover %s" % (stack.stack_name))
discover_stack(options, conn, json_all_stacks, stack.stack_name)
if len(json_all_stacks['stacks']) > 0:
print json.dumps(json_all_stacks, indent=4, sort_keys=True)
def new_stack(options):
consul_conn = Consul(options.host, options.port)
cf_conn = cloudformation.connect_to_region(options.region)
ec2_conn = ec2.connect_to_region(options.region)
stack_validator = NestedStackValidator(cf_conn)
stack_service = StackService(cf_conn, ec2_conn, consul_conn, stack_validator)
return Stack(stack_service, options)
def __init__(self,
environment,
deployment,
region,
zone,
template=template):
# Create connections to AWS components
self.cfn_connection = cfn.connect_to_region(region)
self.sns_connection = sns.connect_to_region(region)
self.vpc_connection = vpc.connect_to_region(region)
self.iam_connection = iam.connect_to_region("universal")
# Temporary python class -> directory name hack
self.lab_dir = self.__class__.__name__.lower()
self.stack_name = "-".join(
[self.lab_dir, environment, deployment, region, zone])
if environment != '':
self.notification_arns = self.get_sns_topic(
"cloudformation-notifications-" + environment)
self.parameters = []
# Prepare the CFN template
self.template_url = "/".join([
os.path.dirname(os.path.realpath(__file__)), self.lab_dir,
vpc_provider, template
])
self.template_body = self.read_file(self.template_url,
max_template_size)
self.validate_template()
def cleanup_stacks():
cfn = cloudformation.connect_to_region("eu-west-1")
cfn.delete_stack("cfn-sphere-test-instances")
wait_for_stack_to_disappear("cfn-sphere-test-instances")
cfn.delete_stack("cfn-sphere-test-vpc")
wait_for_stack_to_disappear("cfn-sphere-test-vpc")
def get_cfn_connection(self):
"""
We persist the CFN connection so that we don't create a new session with each request
"""
if not self.cfn_connection:
self.cfn_connection = cloudformation.connect_to_region(self.config.get('boto').get('region_name'))
return self.cfn_connection
def get_cf_stack(self, stack, resources=False):
"""
Get information on parameters, outputs and resources from a stack and cache it
"""
if not resources:
if not self.cf_stacks.has_key(stack):
#We don't have this stack in the cache already so we need to pull it from CF
cfconn = cloudformation.connect_to_region(self.region)
self.cf_stacks[stack] = cfconn.describe_stacks(stack)[0]
return self.cf_stacks[stack]
else:
if not self.cf_stacks_resources.has_key(stack):
cfconn = cloudformation.connect_to_region(self.region)
the_stack = self.get_cf_stack(stack=stack, resources=False)
self.cf_stacks_resources[stack] = the_stack.list_resources()
return self.cf_stacks_resources[stack]
def get_cfn_connection(self):
"""
We persist the CFN connection so that we don't create a new session with each request
"""
if not self.cfn_connection:
self.cfn_connection = cloudformation.connect_to_region(self.config.get('boto').get('region_name'))
return self.cfn_connection
def get_cf_stack(self, stack, resources = False):
"""
Get information on parameters, outputs and resources from a stack and cache it
"""
if not resources:
if not self.cf_stacks.has_key(stack):
#We don't have this stack in the cache already so we need to pull it from CF
cfconn = cloudformation.connect_to_region(self.region)
self.cf_stacks[stack] = cfconn.describe_stacks(stack)[0]
return self.cf_stacks[stack]
else:
if not self.cf_stacks_resources.has_key(stack):
cfconn = cloudformation.connect_to_region(self.region)
the_stack = self.get_cf_stack(stack = stack, resources = False)
self.cf_stacks_resources[stack] = the_stack.list_resources()
return self.cf_stacks_resources[stack]
def setUpClass(cls):
test_resources_dir = get_resources_dir()
cls.cfn_conn = cloudformation.connect_to_region("eu-west-1")
cls.config = Config(config_file=os.path.join(test_resources_dir, "stacks.yml"))
cls.stack_handler = StackActionHandler(cls.config)
LOGGER.info("Syncing stacks")
cls.stack_handler.create_or_update_stacks()
def upload(region, cfbucket, credentials, stack_name, json_stack_def):
template_key = upload_s3_unique(region, cfbucket, credentials, json_stack_def)
template_url = template_key.generate_url(expires_in=0, query_auth=False)
logger.debug('Template available in s3 at url: %s', template_url)
logger.debug('Uploading stack definition with name: %s', stack_name)
cf = cloudformation.connect_to_region(region, **credentials)
cf.create_stack(stack_name, capabilities=['CAPABILITY_IAM'], template_url=template_url)
logger.debug('Done uploading stack definition')
def destroy_stacks(mmw_config, aws_profile, **kwargs):
"""Destroy stacks that are associated with stack_color"""
region = mmw_config['Region']
stack_color = kwargs['stack_color']
cfn_conn = cfn.connect_to_region(region, profile_name=aws_profile)
color_tag = ('StackColor', stack_color.capitalize())
[stack.delete() for stack in cfn_conn.describe_stacks()
if color_tag in stack.tags.items()]
def delete(self, resource):
if resource.wrapped.stack_status in Stack.VALID_TARGET_STATES:
self.logger.info("Skipping deletion: State '{0}' is a valid target state.".format(
resource.wrapped.stack_status))
return
if self.dry_run:
return
self.logger.info("Initiating deletion sequence for {stack_name}.".format(**vars(resource.wrapped)))
connection = cloudformation.connect_to_region(resource.region)
connection.delete_stack(resource.wrapped.stack_id)
def _get_cf_connection():
conn = cloudformation.connect_to_region(os.environ['AWS_DEFAULT_REGION'])
try:
conn.region
return conn
except AttributeError:
print "Unable to obtain a Cloud Formation connection. Review the " \
"README to ensure you've provided the correct environment to " \
"connect to AWS."
exit()
def get_connection():
try:
connection = connect_to_region(
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
region_name=AWS_CLOUDFORMATION_REGION,
is_secure=True,
)
except BotoServerError, e:
raise Exception("Unable to connect to CloudFormation: %s" %
e.error_message)
def get_connection():
try:
connection = connect_to_region(
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
region_name=AWS_CLOUDFORMATION_REGION,
is_secure=True,
)
except BotoServerError, e:
raise Exception("Unable to connect to CloudFormation: %s"
% e.error_message)
def __init__(self, region="eu-west-1"):
logging.getLogger('boto').setLevel(logging.FATAL)
self.logger = get_logger()
self.conn = cloudformation.connect_to_region(region)
if not self.conn:
self.logger.error("Could not connect to cloudformation API in {0}. Invalid region?".format(region))
raise Exception("Got None connection object")
self.logger.debug("Connected to cloudformation API at {0} with access key id: {1}".format(
region, self.conn.aws_access_key_id))
def __init__(self, profile):
config = AwsConfigMFA()
region = "us-east-1"
r = config.get("profile %s" % profile, "region")
if r != None:
region = r
creds = config.getTokenCredentials(profile)
self.cf = cloudformation.connect_to_region(region,
aws_access_key_id=creds['access_key'],
aws_secret_access_key=creds['secret_key'],
security_token = creds['session_token'])
def wait_for_stack_to_disappear(stack_name):
LOGGER.info("Waiting for stack {0} to disappear".format(stack_name))
cfn = cloudformation.connect_to_region("eu-west-1")
for i in range(1, 30):
stacks = [stack.stack_name for stack in cfn.list_stacks() if stack.stack_status != "DELETE_COMPLETE"]
if stack_name not in stacks:
return
time.sleep(10)
i += 1
raise Exception("Timeout occured waiting for stack {0} to disappear".format(stack_name))
def get_instance_ids_for_stack(stackname):
"""
For a given cloudformation stack, return all of the instance ids, eg
["i-f1430877", "i-g3430877", ...]
"""
instance_ids_for_stack = []
region = cloudformation.connect_to_region(DEFAULT_REGION)
stack_resources = region.describe_stack_resources(stackname)
for stack_resource in stack_resources:
if stack_resource.resource_type == "AWS::EC2::Instance":
instance_ids_for_stack.append(stack_resource.physical_resource_id)
return instance_ids_for_stack
def destroy_stacks(icp_config, aws_profile, **kwargs):
"""Destroy stacks that are associated with stack_color"""
region = icp_config['Region']
stack_type = icp_config['StackType']
stack_color = kwargs['stack_color']
cfn_conn = cfn.connect_to_region(region, profile_name=aws_profile)
stack_tag = ('StackType', stack_type)
color_tag = ('StackColor', stack_color.capitalize())
[stack.delete() for stack in cfn_conn.describe_stacks()
if color_tag in stack.tags.items() and stack_tag in stack.tags.items()]
def get_instance_ids_for_stack(stackname):
"""
For a given cloudformation stack, return all of the instance ids, eg
["i-f1430877", "i-g3430877", ...]
"""
instance_ids_for_stack = []
region = cloudformation.connect_to_region(DEFAULT_REGION)
stack_resources = region.describe_stack_resources(stackname)
for stack_resource in stack_resources:
if stack_resource.resource_type == "AWS::EC2::Instance":
instance_ids_for_stack.append(stack_resource.physical_resource_id)
return instance_ids_for_stack
def fetch_unwanted_resources(self):
for region in self.regions:
connection = cloudformation.connect_to_region(region.name)
unwanted_states = set(connection.valid_states)
unwanted_states.remove("DELETE_COMPLETE")
resources = connection.list_stacks(stack_status_filters=list(unwanted_states)) or []
for resource in resources:
resource_wrapper = Resource(resource, region.name)
if resource.stack_name in self.ignored_resources:
self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
continue
yield resource_wrapper
def delete(self, resource):
if resource.wrapped.stack_status in Stack.VALID_TARGET_STATES:
warnings.warn(
Warning(
"Skipping deletion: State '{0}' is a valid target state.".
format(resource.wrapped.stack_status)))
if self.dry_run:
return
self.logger.info(
"Initiating deletion sequence for {stack_name}.".format(
**vars(resource.wrapped)))
connection = cloudformation.connect_to_region(resource.region)
connection.delete_stack(resource.wrapped.stack_id)
def __init__(self, profile):
config = AwsConfigMFA()
region = "us-east-1"
r = config.get("profile %s" % profile, "region")
if r != None:
region = r
creds = config.getTokenCredentials(profile)
self.cf = cloudformation.connect_to_region(
region,
aws_access_key_id=creds["access_key"],
aws_secret_access_key=creds["secret_key"],
security_token=creds["session_token"],
)
def populate_params(self, current_cf_stacks):
#If we have no parameters in the yaml file, set params to an empty dict and return true
if self.yaml_params is None:
self.params = {}
return True
if self.deps_met(current_cf_stacks):
cfconn = cloudformation.connect_to_region(self.region)
for param in self.yaml_params.keys():
if type(self.yaml_params[param]) is dict:
#Static value set, so use it
if self.yaml_params[param].has_key('value'):
self.params[param] = str(self.yaml_params[param]['value'])
#No static value set, but if we have a source, type and variable can try getting from CF
elif self.yaml_params[param].has_key('source') and self.yaml_params[param].has_key('type') and self.yaml_params[param].has_key('variable'):
if self.yaml_params[param]['source'] == self.mega_stack_name:
source_stack = self.yaml_params[param]['source']
elif self.yaml_params[param]['source'][:1] == '-':
source_stack = self.yaml_params[param]['source'][1:]
else:
source_stack = "%s-%s" % (self.mega_stack_name, self.yaml_params[param]['source'])
self.params[param] = self.get_value_from_cf(
source_stack = source_stack,
var_type = self.yaml_params[param]['type'],
var_name = self.yaml_params[param]['variable']
)
#If self.yaml_params[param] is a list it means there is an array of vars we need to turn into a comma sep list.
elif type(self.yaml_params[param]) is list:
param_list = []
for item in self.yaml_params[param]:
if type(item) is dict:
#Static value set, so use it
if item.has_key('value'):
param_list.append(str(item['value']))
#No static value set, but if we have a source, type and variable can try getting from CF
elif item.has_key('source') and item.has_key('type') and item.has_key('variable'):
if item['source'] == self.mega_stack_name:
source_stack = item['source']
else:
source_stack = "%s-%s" % (self.mega_stack_name, item['source'])
param_list.append(self.get_value_from_cf(
source_stack = source_stack,
var_type = item['type'],
var_name = item['variable']
))
else:
print "Error in yaml file, %s in parameter list for %s stack. Can't populate." % (self.yaml_params[param],self.name)
exit(1)
self.params[param] = ','.join(param_list)
return True
else:
return False
def __init__(self, region="eu-west-1"):
logging.getLogger("boto").setLevel(logging.FATAL)
self.logger = get_logger()
self.conn = cloudformation.connect_to_region(region)
if not self.conn:
self.logger.error(
"Could not connect to cloudformation API in {0}. Invalid region?"
.format(region))
raise Exception("Got None connection object")
self.logger.debug(
"Connected to cloudformation API at {0} with access key id: {1}".
format(region, self.conn.aws_access_key_id))
def _query_stack(options):
if options["profile"] != "":
conn = cloudformation.connect_to_region(options["region"], profile_name=options["profile"])
else:
conn = cloudformation.connect_to_region(
options["region"],
aws_access_key_id=options["AWS_ACCESS_KEY_ID"],
aws_secret_access_key=options["AWS_SECRET_ACCESS_KEY"],
)
stack_status_filters = ["CREATE_IN_PROGRESS", "CREATE_COMPLETE", "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE"]
next_token = None
end_list = False
json_all_stacks = dict()
json_all_stacks["stacks"] = dict()
## Using no regex will improve speed of the process
if options["regex"] == False:
for stack_name in options["stacks"]:
debug("Will discover %s" % (stack_name))
discover_stack(options, conn, json_all_stacks, stack_name)
else:
while end_list == False:
stacks = conn.list_stacks(stack_status_filters=stack_status_filters, next_token=next_token)
next_token = stacks.next_token
if next_token == None:
end_list = True
for stack in stacks:
debug("%s : %s" % (stack.stack_name, stack.stack_status))
for match_stack in options["stacks"]:
a = re.compile(match_stack)
result = a.match(stack.stack_name)
if result == None:
continue
debug("Will discover %s" % (stack.stack_name))
discover_stack(options, conn, json_all_stacks, stack.stack_name)
if len(json_all_stacks["stacks"]) > 0:
print json.dumps(json_all_stacks, indent=4, sort_keys=True)
def populate_params(self, current_cf_stacks):
#If we have no parameters in the yaml file, set params to an empty dict and return true
if self.yaml_params is None:
self.params = {}
return True
if self.deps_met(current_cf_stacks):
cfconn = cloudformation.connect_to_region(self.region)
for param in self.yaml_params.keys():
if type(self.yaml_params[param]) is dict:
#Static value set, so use it
if self.yaml_params[param].has_key('value'):
self.params[param] = str(self.yaml_params[param]['value'])
#No static value set, but if we have a source, type and variable can try getting from CF
elif self.yaml_params[param].has_key('source') and self.yaml_params[param].has_key('type') and self.yaml_params[param].has_key('variable'):
if self.yaml_params[param]['source'] == self.mega_stack_name:
source_stack = self.yaml_params[param]['source']
else:
source_stack = "%s-%s" % (self.mega_stack_name, self.yaml_params[param]['source'])
self.params[param] = self.get_value_from_cf(
source_stack = source_stack,
var_type = self.yaml_params[param]['type'],
var_name = self.yaml_params[param]['variable']
)
#If self.yaml_params[param] is a list it means there is an array of vars we need to turn into a comma sep list.
elif type(self.yaml_params[param]) is list:
param_list = []
for item in self.yaml_params[param]:
if type(item) is dict:
#Static value set, so use it
if item.has_key('value'):
param_list.append(str(item['value']))
#No static value set, but if we have a source, type and variable can try getting from CF
elif item.has_key('source') and item.has_key('type') and item.has_key('variable'):
if item['source'] == self.mega_stack_name:
source_stack = item['source']
else:
source_stack = "%s-%s" % (self.mega_stack_name, item['source'])
param_list.append(self.get_value_from_cf(
source_stack = source_stack,
var_type = item['type'],
var_name = item['variable']
))
else:
print "Error in yaml file, %s in parameter list for %s stack. Can't populate." % (self.yaml_params[param],self.name)
exit(1)
self.params[param] = ','.join(param_list)
return True
else:
return False
def __init__(self, aws_access_key_id, aws_secret_access_key,
security_token):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.security_token = security_token
self.cf = cloudformation.connect_to_region(
region_name='us-west-2',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
security_token=security_token)
self.ec2 = ec2.connect_to_region(
'us-west-2',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
security_token=security_token)
def connect_cloudformation():
""" Connect to AWS CloudFormation
:returns: boto.cloudformation.connection
"""
try:
return cloudformation.connect_to_region(
config.get_environment_option('region'),
aws_access_key_id=config.get_environment_option('access-key-id'),
aws_secret_access_key=config.get_environment_option(
'secret-access-key'))
except Exception as err:
logger.error(
'A problem occurred connecting to AWS CloudFormation: {}'.format(
err))
raise
def connect_cloudformation():
""" Connect to AWS CloudFormation
:returns: boto.cloudformation.connection
"""
try:
return cloudformation.connect_to_region(
config.get_environment_option('region'),
aws_access_key_id=config.get_environment_option(
'access-key-id'),
aws_secret_access_key=config.get_environment_option(
'secret-access-key'))
except Exception as err:
logger.error(
'A problem occurred connecting to AWS CloudFormation: {}'.format(
err))
raise
def __init__(self, environment, deployment, region, zone,
aws_access_key_id, aws_secret_access_key):
# Create connections to AWS components
self.cfn_connection = cfn.connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
self.sns_connection = sns.connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
self.vpc_connection = vpc.connect_to_region(region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
# Temporary python class -> directory name hack
lab_dir = self.__class__.__name__.lower()
self.stack_name = "-".join([lab_dir, environment, deployment, region, zone])
self.notification_arns = self.get_sns_topic("cloudformation-notifications-" + environment)
self.parameters = []
# Prepare the CFN template
self.template_url = "/".join([os.path.dirname(os.path.realpath(__file__)), lab_dir, vpc_provider, template])
self.template_body = self.read_file(self.template_url, max_template_size)
self.validate_template()
def get_aws_connection(service, region_name='eu-west-1'):
credentials = {
'aws_access_key_id': config.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': config.AWS_SECERET_ACCESS_KEY
}
if service == 'ec2':
return ec2.connect_to_region(region_name, **credentials)
elif service == 'cloudformation':
return cloudformation.connect_to_region(region_name, **credentials)
elif service == 'autoscale':
return autoscale.connect_to_region(region_name, **credentials)
elif service == 'elb':
return elb.connect_to_region(region_name, **credentials)
elif service == 'cloudwatch':
return cloudwatch.connect_to_region(region_name, **credentials)
else:
raise Exception("Unkown service '%s'" % service)
def fetch_unwanted_resources(self):
for region_name in self.region_names:
connection = cloudformation.connect_to_region(region_name)
unwanted_states = set(connection.valid_states)
unwanted_states.remove("DELETE_COMPLETE")
resources = connection.list_stacks(
stack_status_filters=list(unwanted_states)) or []
for resource in resources:
resource_wrapper = Resource(
resource=resource,
resource_type=self.resource_type,
resource_id=resource.stack_id,
creation_date=resource.creation_time,
region=region_name)
if resource.stack_name in self.ignored_resources:
self.logger.info('IGNORE ' +
self.to_string(resource_wrapper))
continue
yield resource_wrapper
def __init__(self, module, env, cluster, zone, template_name='template.json', **optionals):
# debug and/or dry-run mode
self.debug = optionals['debug']
self.dry_run = optionals['dry_run']
self.enable_debug()
self.env = env.strip().lower()
self.cluster = snake_case_with_dashes(cluster.strip())
self.zone = zone
self.template_name = template_name
# for stack names & resources names in template, use only non-default names
self.non_default_cluster = self.cluster if self.cluster not in [default_cluster_name] else None
# stack name
self.module_name = snake_case_with_dashes(module.strip())
self.stack_name = "-".join(filter(None, [self.module_name, self.env, self.non_default_cluster, self.zone]))
# params
self.parameters = []
# stack tags
self.stack_tags = {
'{0}:minion:env'.format(org_name): self.env,
'{0}:minion:cluster'.format(org_name): self.cluster,
'{0}:minion:module'.format(org_name): self.module_name
}
# Create connections to AWS components
aws_access_key_id = get_env_variable('MINION_ACCESS_KEY_ID')
aws_secret_access_key = get_env_variable('MINION_SECRET_ACCESS_KEY')
self.cfn_connection = cfn.connect_to_region(aws_region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
self.vpc_connection = vpc.connect_to_region(aws_region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
self.iam_connection = iam.connect_to_region("universal",
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
def get_cfn_conn():
# type: () -> CloudFormationConnection
"""Creates CloudFormation connection object using region data in playbooks/vars/main.yaml"""
# Open vars/main.yaml. Used to get AWS region
with open('playbooks/vars/main.yaml', mode='r') as f:
try:
vars = yaml.load(f)
except yaml.YAMLError as e:
print(
"There was a problem loading 'playbooks/vars/main.yaml. \n%s" %
e.message)
exit(1)
# Get CloudFormation connection
try:
cfn_conn = connect_to_region(region_name=vars['region'])
except Exception as e:
print("There was a problem connecting to AWS. \n%s" % e.message)
exit(1)
return cfn_conn
def wait_until_stack_create_complete(stackname):
"""
Wait until this stack is complete. In other words, when there is a stack event
with:
event.resource_type = AWS::CloudFormation::Stack
event.resource_status = CREATE_COMPLETE
"""
for x in xrange(NUM_RETRIES):
print ("Waiting for {} to finish launching. Attempt: {}".format(stackname, x))
region = cloudformation.connect_to_region(DEFAULT_REGION)
try:
region.describe_stacks(stackname)
except BotoServerError as bse:
print ("Exception describing stack: {}, exception: {}. Retrying.".format(stackname, bse))
continue
stack_events = region.describe_stack_events(stackname)
for stack_event in stack_events:
if stack_event.stack_name != stackname:
print (
"Ignoring {} since it's stack name is {} instead of {}".format(
stack_event, stack_event.stack_name, stackname
)
)
continue
if (
stack_event.resource_type == "AWS::CloudFormation::Stack"
and stack_event.resource_status == "CREATE_COMPLETE"
):
print ("Stack {} has successfully been created".format(stackname))
return
# didn't find it, lets wait and try again
time.sleep(5)
def _launch_cfn(self):
"""
Sets up stack and launches it.
"""
self.set_up_stack()
self.boto_conn = cloudformation.connect_to_region(region_name=self.aws_region,
profile_name=self.aws_profile)
parameters = []
for param, input_name in self.input_wiring.iteritems():
try:
parameters.append((param, self.get_input(input_name)))
except MKInputError:
pass
# check to see if stack exists
try:
self.stack = self.boto_conn.describe_stacks(self.stack_name)[0]
except BotoServerError:
# it would be great if we could more granularly check the error
self.boto_conn.create_stack(self.stack_name,
tags=self.get_raw_tags(),
template_body=self.to_json(),
parameters=parameters)
self.logger.info('Stack %s created', self.stack_name)
def get_value_from_cf(self, source_stack, var_type, var_name):
"""
Get a variable from a existing cloudformation stack, var_type should be parameter, resource or output.
If using resource, provide the logical ID and this will return the Physical ID
"""
cfconn = cloudformation.connect_to_region(self.region)
the_stack = self.get_cf_stack(stack = source_stack)
if var_type == 'parameter':
for p in the_stack.parameters:
if str(p.key) == var_name:
return str(p.value)
elif var_type == 'output':
for o in the_stack.outputs:
if str(o.key) == var_name:
return str(o.value)
elif var_type == 'resource':
for r in self.get_cf_stack(stack = source_stack, resources = True):
if str(r.logical_resource_id) == var_name:
return str(r.physical_resource_id)
else:
print "Error: invalid var_type passed to get_value_from_cf, needs to be parameter, resource or output. Not: %s" % (var_type)
exit(1)
def get_value_from_cf(self, source_stack, var_type, var_name):
"""
Get a variable from a existing cloudformation stack, var_type should be parameter, resource or output.
If using resource, provide the logical ID and this will return the Physical ID
"""
cfconn = cloudformation.connect_to_region(self.region)
the_stack = self.get_cf_stack(stack = source_stack)
if var_type == 'parameter':
for p in the_stack.parameters:
if str(p.key) == var_name:
return str(p.value)
elif var_type == 'output':
for o in the_stack.outputs:
if str(o.key) == var_name:
return str(o.key)
elif var_type == 'resource':
for r in self.get_cf_stack(stack = source_stack, resources = True):
if str(r.logical_resource_id) == var_name:
return str(r.physical_resource_id)
else:
print "Error: invalid var_type passed to get_value_from_cf, needs to be parameter, resource or output. Not: %s" % (var_type)
exit(1)
def wait_until_stack_create_complete(stackname):
"""
Wait until this stack is complete. In other words, when there is a stack event
with:
event.resource_type = AWS::CloudFormation::Stack
event.resource_status = CREATE_COMPLETE
"""
for x in xrange(NUM_RETRIES):
print("Waiting for {} to finish launching. Attempt: {}".format(
stackname, x))
region = cloudformation.connect_to_region(DEFAULT_REGION)
try:
region.describe_stacks(stackname)
except BotoServerError as bse:
print("Exception describing stack: {}, exception: {}. Retrying.".
format(stackname, bse))
continue
stack_events = region.describe_stack_events(stackname)
for stack_event in stack_events:
if stack_event.stack_name != stackname:
print("Ignoring {} since it's stack name is {} instead of {}".
format(stack_event, stack_event.stack_name, stackname))
continue
if stack_event.resource_type == "AWS::CloudFormation::Stack":
if stack_event.resource_status in [
"CREATE_COMPLETE", "UPDATE_COMPLETE"
]:
print("Stack {} has successfully been created/updated".
format(stackname))
return
# didn't find it, lets wait and try again
time.sleep(5)
def _get_connection(profile_name, region_name):
LOGGER.info(
'Connecting to AWS using Profile: {profile} in Region {region}'.format(
profile=profile_name, region=region_name))
return cloudformation.connect_to_region(region_name=region_name,
profile_name=profile_name)
def __init__(self, yamlFile):
self.logger = logging.getLogger(__name__)
#load the yaml file and turn it into a dict
thefile = open(yamlFile, 'r')
self.stackDict = yaml.safe_load(thefile)
#Make sure there is only one top level element in the yaml file
if len(self.stackDict.keys()) != 1:
self.logger.critical(
"Need one and only one mega stack name at the top level, found %s"
% len(self.stackDict.keys()))
exit(1)
#How we know we only have one top element, that must be the mega stack name
self.name = self.stackDict.keys()[0]
#Find and set the mega stacks region. Exit if we can't find it
if self.stackDict[self.name].has_key('region'):
self.region = self.stackDict[self.name]['region']
else:
self.logger.critical(
"No region specified for mega stack, don't know where to build it."
)
exit(1)
self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', [])
if isinstance(self.sns_topic_arn, str):
self.sns_topic_arn = [self.sns_topic_arn]
for topic in self.sns_topic_arn:
if topic.split(':')[3] != self.region:
self.logger.critical("SNS Topic %s is not in the %s region." %
(topic, self.region))
exit(1)
#Array for holding CFStack objects once we create them
self.stack_objs = []
#Get the names of the sub stacks from the yaml file and sort in array
self.cf_stacks = self.stackDict[self.name]['stacks'].keys()
#Megastack holds the connection to cloudformation and list of stacks currently in our region
#Stops us making lots of calls to cloudformation API for each stack
try:
self.cfconn = cloudformation.connect_to_region(self.region)
self.cf_desc_stacks = self.cfconn.describe_stacks()
except boto.exception.NoAuthHandlerFound as e:
self.logger.critical(
"No credentials found for connecting to cloudformation: %s" %
e)
exit(1)
#iterate through the stacks in the yaml file and create CFstack objects for them
for stack_name in self.cf_stacks:
the_stack = self.stackDict[self.name]['stacks'][stack_name]
if type(the_stack) is dict:
local_sns_arn = the_stack.get('sns-topic-arn',
self.sns_topic_arn)
if isinstance(local_sns_arn, str):
local_sns_arn = [local_sns_arn]
for topic in local_sns_arn:
if topic.split(':')[3] != self.region:
self.logger.critical(
"SNS Topic %s is not in the %s region." %
(topic, self.region))
exit(1)
if the_stack.has_key('cf_template'):
self.stack_objs.append(
CFStack(mega_stack_name=self.name,
name=stack_name,
params=the_stack['params'],
template_name=the_stack['cf_template'],
region=self.region,
sns_topic_arn=local_sns_arn,
depends_on=the_stack['depends']))
elif opt in ("-y"):
options['noconfirmation'] = True
elif opt in ("-s", "--stack"):
try:
re.compile(arg)
except Exception, e:
logger.error("Invalid regular expression %s", arg)
sys.exit(1)
options['stacks'].append(arg)
if not options['region']:
logger.error('Needs region specified.')
sys.exit(1)
if options['profile']:
conn = cloudformation.connect_to_region(options['region'], profile_name=options['profile'])
elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ:
conn = cloudformation.connect_to_region(
options['region'],
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY']
)
else:
logger.error('Missing AWS credentials. Check your environment/profile.')
sys.exit(1)
if options['list_stacks']:
for s in get_safe_stack_list(conn, options['stacks']):
print s.stack_name
sys.exit()
if __name__ == '__main__':
args = docopt(__doc__, version='Lambda Chat AWS Resources 0.2')
config = load_config()
print_cf_template = args['cf'] or args['launch']
try:
if print_cf_template:
template = generate_cf_template()
print(template.to_json())
if (args['cf']):
sys.exit(1)
# Get a connection to AWS CloudFormation in the given region
conn = cloudformation.connect_to_region(args['--region'])
if (args['launch']):
launch(args, config, conn, template)
elif (args['update']):
update(args, config, conn, template)
elif (args['delete']):
delete(args, config, conn)
elif (args['output']):
output(args, config, conn)
except Exception, e:
def __init__(self, yamlFile):
self.logger = logging.getLogger(__name__)
#load the yaml file and turn it into a dict
thefile = open(yamlFile, 'r')
self.stackDict = yaml.safe_load(thefile)
#Make sure there is only one top level element in the yaml file
if len(self.stackDict.keys()) != 1:
self.logger.critical("Need one and only one mega stack name at the top level, found %s" % len(self.stackDict.keys()))
exit(1)
#How we know we only have one top element, that must be the mega stack name
self.name = self.stackDict.keys()[0]
#Find and set the mega stacks region. Exit if we can't find it
if self.stackDict[self.name].has_key('region'):
self.region = self.stackDict[self.name]['region']
else:
self.logger.critical("No region specified for mega stack, don't know where to build it.")
exit(1)
self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', [])
if isinstance(self.sns_topic_arn, str): self.sns_topic_arn = [self.sns_topic_arn]
for topic in self.sns_topic_arn:
if topic.split(':')[3] != self.region:
self.logger.critical("SNS Topic %s is not in the %s region." % (topic, self.region))
exit(1)
#Array for holding CFStack objects once we create them
self.stack_objs = []
#Get the names of the sub stacks from the yaml file and sort in array
self.cf_stacks = self.stackDict[self.name]['stacks'].keys()
#Megastack holds the connection to cloudformation and list of stacks currently in our region
#Stops us making lots of calls to cloudformation API for each stack
try:
self.cfconn = cloudformation.connect_to_region(self.region)
self.cf_desc_stacks = self.cfconn.describe_stacks()
except boto.exception.NoAuthHandlerFound as e:
self.logger.critical("No credentials found for connecting to cloudformation: %s" % e )
exit(1)
#iterate through the stacks in the yaml file and create CFstack objects for them
for stack_name in self.cf_stacks:
the_stack = self.stackDict[self.name]['stacks'][stack_name]
if type(the_stack) is dict:
local_sns_arn = the_stack.get('sns-topic-arn', self.sns_topic_arn)
if isinstance(local_sns_arn, str): local_sns_arn = [local_sns_arn]
for topic in local_sns_arn:
if topic.split(':')[3] != self.region:
self.logger.critical("SNS Topic %s is not in the %s region." % (topic, self.region))
exit(1)
if the_stack.has_key('cf_template'):
self.stack_objs.append(
CFStack(
mega_stack_name=self.name,
name=stack_name,
params=the_stack['params'],
template_name=the_stack['cf_template'],
region=self.region,
sns_topic_arn=local_sns_arn,
depends_on=the_stack['depends']
)
)
def cloudformation(self):
if not hasattr(self, '_cloudformation'):
self._cloudformation = cloudformation.connect_to_region(
self.region)
return self._cloudformation
print ("Stack creation failed.")
exit()
# Deleting the older stack
def delete_stack(connection_cf,stack_name):
stack_list=connection_cf.list_stacks(stack_status_filters=['CREATE_COMPLETE'])
for stack in stack_list:
if (stack.stack_name).startswith('T3Hydration'):
if stack.stack_name != stack_name:
try:
delete_stack(connection_cf,stack.stack_name)
except Exception as e:
print "Error occurred while deleting the stack."
print e
# Creating a cloud formation stack ends here.
# Main functions starts here.
connection_ec2 = ec2.connect_to_region(aws_region,aws_access_key_id=AWS_ACCESS_KEY_ID,aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
connection_cf = cf.connect_to_region(aws_region,aws_access_key_id=AWS_ACCESS_KEY_ID,aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
timestr = time.strftime("%Y%m%d")
stack_name="T3Hydration-"+str(timestr)
create_snapshots(connection_ec2)
get_latest_ami_id(connection_ec2)
update_cf_template()
with open(file_name,'r') as f:
tags = {'Name':stack_name}
create_stack(connection_cf,stack_name,f.read(),tags,[])
#delete_stack(connection_cf,stack_name)
def __init__(self, yamlFile):
self.logger = logging.getLogger(__name__)
# load the yaml file and turn it into a dict
thefile = open(yamlFile, 'r')
rendered_file = pystache.render(thefile.read(), dict(os.environ))
self.stackDict = yaml.safe_load(rendered_file)
# Make sure there is only one top level element in the yaml file
if len(self.stackDict.keys()) != 1:
error_message = ("Need one and only one mega stack name at the"
+ " top level, found %s")
self.logger.critical(error_message % len(self.stackDict.keys()))
exit(1)
# Now we know we only have one top element,
# that must be the mega stack name
self.name = self.stackDict.keys()[0]
# Find and set the mega stacks region. Exit if we can't find it
if 'region' in self.stackDict[self.name]:
self.region = self.stackDict[self.name]['region']
else:
self.logger.critical("No region specified for mega stack,"
+ " don't know where to build it.")
exit(1)
if 'account_id' in self.stackDict[self.name]:
# Get the account ID for the current AWS credentials
iamconn = iam.connect_to_region(self.region)
user_response = iamconn.get_user()['get_user_response']
user_result = user_response['get_user_result']
account_id = user_result['user']['arn'].split(':')[4]
# Check if the current account ID matches the stack's account ID
if account_id != str(self.stackDict[self.name]['account_id']):
self.logger.critical("Account ID of stack does not match the"
+ " account ID of your AWS credentials.")
exit(1)
self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', [])
if isinstance(self.sns_topic_arn, str):
self.sns_topic_arn = [self.sns_topic_arn]
for topic in self.sns_topic_arn:
if topic.split(':')[3] != self.region:
self.logger.critical("SNS Topic %s is not in the %s region."
% (topic, self.region))
exit(1)
self.global_tags = self.stackDict[self.name].get('tags', {})
# Array for holding CFStack objects once we create them
self.stack_objs = []
# Get the names of the sub stacks from the yaml file and sort in array
self.cf_stacks = self.stackDict[self.name]['stacks'].keys()
# Megastack holds the connection to CloudFormation and list of stacks
# currently in our region stops us making lots of calls to
# CloudFormation API for each stack
try:
self.cfconn = cloudformation.connect_to_region(self.region)
self.cf_desc_stacks = self._describe_all_stacks()
except boto.exception.NoAuthHandlerFound as exception:
self.logger.critical(
"No credentials found for connecting to CloudFormation: %s"
% exception)
exit(1)
# iterate through the stacks in the yaml file and create CFstack
# objects for them
for stack_name in self.cf_stacks:
the_stack = self.stackDict[self.name]['stacks'][stack_name]
if type(the_stack) is dict:
if the_stack.get('disable', False):
warn_message = ("Stack %s is disabled by configuration"
+ " directive. Skipping")
self.logger.warning(warn_message % stack_name)
continue
local_sns_arn = the_stack.get('sns-topic-arn',
self.sns_topic_arn)
if isinstance(local_sns_arn, str):
local_sns_arn = [local_sns_arn]
for topic in local_sns_arn:
if topic.split(':')[3] != self.region:
error_message = "SNS Topic %s is not in the %s region."
self.logger.critical(error_message
% (topic, self.region))
exit(1)
local_tags = the_stack.get('tags', {})
merged_tags = dict(self.global_tags.items()
+ local_tags.items())
# Add static cumulus-stack tag
merged_tags['cumulus-stack'] = self.name
if 'cf_template' in the_stack:
self.stack_objs.append(
CFStack(
mega_stack_name=self.name,
name=stack_name,
params=the_stack.get('params'),
template_name=the_stack['cf_template'],
region=self.region,
sns_topic_arn=local_sns_arn,
depends_on=the_stack.get('depends'),
tags=merged_tags
)
)
def __init__(self, region, profile):
self.region = region
self.cf = cf.connect_to_region(self.region, profile_name=profile)
def cloudformation(self):
if not hasattr(self, '_cloudformation'):
self._cloudformation = cloudformation.connect_to_region(
self.region)
return self._cloudformation
if __name__ == '__main__':
args = docopt(__doc__, version='Lambda Chat AWS Resources 0.2')
config = load_config()
print_cf_template = args['cf'] or args['launch']
try:
if print_cf_template:
template = generate_cf_template()
print(template.to_json())
if (args['cf']):
sys.exit(1)
# Get a connection to AWS CloudFormation in the given region
conn = cloudformation.connect_to_region(
args['--region'], profile_name=args['--profile'])
if (args['launch']):
launch(args, config, conn, template)
elif (args['update']):
update(args, config, conn, template)
elif (args['delete']):
delete(args, config, conn)
elif (args['output']):
output(args, config, conn)
except Exception, e:
def __init__(self, aws_access_key_id, aws_secret_access_key, security_token):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.security_token = security_token
self.cf = cloudformation.connect_to_region(region_name = 'us-west-2', aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, security_token = security_token)
self.ec2 = ec2.connect_to_region('us-west-2', aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key, security_token = security_token)
def __init__(self):
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
self.test_resources_dir = self._get_resources_dir()
self.cfn_conn = cloudformation.connect_to_region("eu-west-1")
self.config = Config(config_file=os.path.join(self.test_resources_dir, "stacks.yml"))
