def get_change_set(self, change_set_name, change_set_description,
change_set_type):
# create the change set
if self.stack_status:
resp = self.client.create_change_set(
StackName=self.stack_name,
TemplateURL=self.template_url,
Parameters=self.build_params(),
Capabilities=[
'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM',
'CAPABILITY_AUTO_EXPAND'
],
ChangeSetName=change_set_name,
Description=change_set_description,
ChangeSetType=change_set_type)
logger.info("Change Set Started: %s" % resp['Id'])
sleep(5)
self.change_set_status = self.reload_change_set_status(
change_set_name)
while self.change_set_status != 'CREATE_COMPLETE':
sleep(10)
status = self.reload_change_set_status(change_set_name)
logger.info(status)
if status == 'FAILED':
raise RuntimeError("Change set Failed")
self.print_change_set(change_set_name, change_set_description)
else:
raise RuntimeError("Stack does not exist")
def update_stack(self):
# update the stack
waiter = self.client.get_waiter('stack_update_complete')
start_time = datetime.now(pytz.utc)
args = {
"StackName":
self.stack_name,
"Parameters":
self.build_params(),
"Tags":
self.construct_tags(),
"Capabilities": [
'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM',
'CAPABILITY_AUTO_EXPAND'
]
}
args.update({'TemplateBody': self.template_body} if self.
template_body else {"TemplateURL": self.template_url})
if self.template_body:
logger.info("Using local template due to null template bucket")
if self.stack_status:
try:
self.client.update_stack(**args)
self.update_waiter(start_time)
except ClientError as e:
if 'No updates are to be performed' in e.response['Error'][
'Message']:
logger.warning('No updates are to be performed')
else:
raise e
else:
raise RuntimeError("Stack does not exist")
def skip_or_send(self, fname, dest_key):
try:
etag = self.generate_etag(fname)
self.client.get_object(Bucket=self.dest_bucket,
IfMatch=etag,
Key=dest_key)
logger.debug("Skipped: %s" % (fname))
except Exception as e:
self.client.upload_file(fname, self.dest_bucket, dest_key)
logger.info("Uploaded: %s to s3://%s/%s" %
(fname, self.dest_bucket, dest_key))
def zip_lambdas(self):
logger.info('Creating Lambda Archives')
if self.lambda_dirs:
for dir in self.lambda_dirs:
logger.info('Archiving ' + dir)
if os.path.exists(dir):
temp_dir = dir + "_temp"
logger.debug('Creating ' + temp_dir)
shutil.copytree(dir, temp_dir)
if os.path.exists("/".join([temp_dir, "package.json"])):
req_txt = temp_dir + "/package.json"
logger.debug('Found ' + req_txt)
# Nodejs
subprocess.call(["npm", "install"], cwd=temp_dir)
subprocess.call(["npm", "run", "build"], cwd=temp_dir)
if os.path.exists("/".join([temp_dir,
"requirements.txt"])):
req_txt = "/".join([temp_dir, "requirements.txt"])
logger.debug('Found ' + req_txt)
try:
# Python 3
subprocess.run([
"pip", "install", "-q", "-r", req_txt, "-t",
temp_dir
])
except AttributeError:
# Python 2
subprocess.call([
"pip", "install", "-q", "-r", req_txt, "-t",
temp_dir
])
logger.debug('Archiving ' + dir.split('/')[-1])
shutil.make_archive(dir.split('/')[-1], "zip", temp_dir)
logger.debug('Removing ' + temp_dir)
shutil.rmtree(temp_dir)
file_name = "{}.zip".format(dir.split('/')[-1])
dest = '/'.join(
[self.sync_base,
'/'.join(dir.split('/')[:-1])]).replace('//', '/')
if not os.path.exists(dest): os.mkdir(dest)
logger.debug('Moving archive to ' + dest)
shutil.copy(file_name, dest)
os.remove(file_name)
else:
raise ValueError(
"Lambda path '{}' does not exist.".format(dir))
else:
logger.debug("No 'lambda_dirs' defined in stack; ignoring -z flag")
def test(self):
logger.info("Validating Templates")
if self.sync_dirs:
count = 0
processes = []
for sync_dir in self.sync_dirs:
sync_dir = sync_dir.strip(".")
for dirName, subdirList, fileList in os.walk(
"%s%s" % (self.base, sync_dir)):
thisdir = "".join(dirName.rsplit(self.base)).strip("/")
fileList = [
os.path.join(dirName, filename)
for filename in fileList
]
if self.excludes:
for ignore in self.excludes:
fileList = [
n for n in fileList
if not fnmatch.fnmatch(n, ignore)
]
for fname in fileList:
dest_key = self.generate_dest_key(fname, thisdir)
if os.name != 'nt':
processes.append(
Process(target=self.validate,
args=(fname, dest_key)))
processes[count].deamon = True
processes[count].start()
if count % 5 == 0:
processes[count].join()
else:
self.validate(fname, dest_key)
count += 1
if len(processes) != 0:
for process in processes:
process.join()
for process in processes:
if process.exitcode:
logger.critical(
"Failed to validate templates before upload")
exit(1)
def update_waiter(self, start_time):
waiter = self.client.get_waiter('stack_update_complete')
logger.info("Update Started")
sleep(5)
logger.info(self.reload_stack_status())
if self.print_events:
self.output_events(start_time, 'update')
else:
try:
waiter.wait(StackName=self.stack_name)
except WaiterError as e:
status = self.reload_stack_status()
logger.info(status)
self.output_events(start_time, 'update')
logger.info(self.reload_stack_status())
def create_stack(self):
# create the stack
start_time = datetime.now(pytz.utc)
args = {
"StackName":
self.stack_name,
"Parameters":
self.build_params(),
"DisableRollback":
self.disable_rollback,
"Tags":
self.construct_tags(),
"Capabilities": [
'CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM',
'CAPABILITY_AUTO_EXPAND'
]
}
args.update({'TemplateBody': self.template_body} if self.
template_body else {"TemplateURL": self.template_url})
if self.template_body:
logger.info("Using local template due to null template bucket")
resp = self.client.create_stack(**args)
self.create_waiter(start_time)
def build_params(self):
# create parameters from the config.yml file
self.parameter_file = "%s-params.json" % self.stack
expanded_params = []
expanded_params.append({
"ParameterKey": "Release",
"ParameterValue": self.release
})
# Order of the stacks is priority on overwrites, authoritative is last
# Here we loop through all of the params in the config file, we need to
# create a array of parameter objects, we have to loop through our array
# to ensure we dont already have one of that key.
for env in ['global', self.stack]:
if 'parameters' in self.config[env]:
logger.debug("env {0} has parameters: {1}".format(
env, self.config[env]['parameters']))
for param_key, param_value in self.config[env][
'parameters'].items():
count = 0
overwritten = False
param_xform = ','.join(param_value) if isinstance(
param_value, list) else param_value
for param_item in expanded_params:
if param_item['ParameterKey'] == param_key:
expanded_params[count] = {
"ParameterKey": param_key,
"ParameterValue": param_xform
}
overwritten = True
count += 1
if not overwritten:
expanded_params.append({
"ParameterKey": param_key,
"ParameterValue": param_xform
})
if 'lookup_parameters' in self.config[env]:
for param_key, lookup_struct in self.config[env][
'lookup_parameters'].items():
stack = Stack(self.profile, self.config_file,
lookup_struct['Stack'])
stack.get_outputs()
for output in stack.outputs:
if output['OutputKey'] == lookup_struct['OutputKey']:
expanded_params.append({
"ParameterKey":
param_key,
"ParameterValue":
output['OutputValue']
})
# Remove overridden parameters and set them based on the override
# provided. Explicit overrides take priority over anything in the
# configuration files.
expanded_params = [
x for x in expanded_params
if x['ParameterKey'] not in self.params.keys()
]
expanded_params += [{
"ParameterKey": x,
"ParameterValue": self.params[x]
} for x in self.params.keys()]
# Here we restrict the returned parameters to only the ones that the
# template accepts by copying expanded_params into return_params and removing
# the item in question from return_params
logger.debug("expanded_params: {0}".format(expanded_params))
return_params = list(expanded_params)
with open(self.template_file, 'r') as template_file:
if re.match(".*\.json", self.template_file):
parsed_template_file = json.load(template_file)
elif re.match(".*\.ya?ml", self.template_file):
parsed_template_file = ruamel.yaml.safe_load(template_file)
else:
logger.info("Filename does not end in json/yml/yaml")
return return_params
for item in expanded_params:
logger.debug("item: {0}".format(item))
if item['ParameterKey'] not in parsed_template_file.get(
'Parameters', {}):
logger.debug(
"Not using parameter '{0}': not found in template '{1}'"
.format(item['ParameterKey'], template_file))
return_params.remove(item)
logger.info("Parameters Created")
return return_params
def delete_stack(self):
logger.info("Sent delete request to stack")
resp = self.client.delete_stack(StackName=self.stack_name)
return True