def test_skip_ecs_changeset(self):
changeset = ParentTestCase._get_resource_path("cloudformation/sample_changeset.json")
cloudformation = CloudFormationBuddy(self.test_deploy_ctx)
with open(changeset, 'r') as cs:
cloudformation.change_set_description = json.load(cs)
cloudformation.existing_change_set_id = cloudformation.change_set_description['ChangeSetId']
self.assertFalse(cloudformation.should_execute_change_set(),"Failed to skip ecs special case")
def test_changeset_operation_ready(self):
cloudformation = CloudFormationBuddy(self.test_deploy_ctx)
try:
cloudformation.describe_change_set()
self.fail("Failed to throw error when not ready for decribe changeset")
except:
pass
def do_command(deploy_ctx, type_filter):
# type: (DeployContext,str) -> None
cf_buddy = CloudFormationBuddy(deploy_ctx=deploy_ctx)
stacks = cf_buddy.list_stacks(deploy_ctx.stack_name)
resources = cf_buddy.load_resources_for_stack_list(stacks)
for stack_name, resources in resources.items():
print_utility.banner("Stack: {}".format(stack_name))
for resource in resources:
if not type_filter or type_filter in resource['ResourceType']:
print_utility.info_banner("\tName: {}".format(
resource['LogicalResourceId']))
print_utility.info_banner("\tType: {}".format(
resource['ResourceType']))
def test_cloudformation_deploy(self):
template = ParentTestCase._get_resource_path("cloudformation/aws-resources.template")
parameter_file = ParentTestCase._get_resource_path("cloudformation/aws-resources.parameters.json")
config_templates = ParentTestCase._get_resource_path("cloudformation/config/")
deploy = CloudFormationDeploy(self.test_deploy_ctx.stack_name, LocalTemplate(template, parameter_file, config_templates), self.test_deploy_ctx)
deploy.do_deploy(dry_run=False)
cloudformation = CloudFormationBuddy(self.test_deploy_ctx)
s3 = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
try:
self.assertTrue(cloudformation.does_stack_exist(), "Failed to create stack")
self.assertEqual(s3.get_file_as_string("install_template.sh"),"foo-bar-{}".format(self.run_random_word),"Did not render config template")
finally:
super(CloudFormationTestCase, self).clean(cloudformation)
super(CloudFormationTestCase, self).clean_s3(s3)
def test_helper_funcs(self):
ctx = DeployContext.create_deploy_context(application="dev-{}".format(
self.run_random_word),
role="cluster",
environment="unit-test",
defaults=self.default_config)
cloudformation = CloudFormationBuddy(ctx)
try:
template_dir = ParentTestCase._get_resource_path(
"parameter_load_tests/helper_func")
deploy = CloudFormationDeploy(ctx.stack_name,
NamedLocalTemplate(template_dir),
ctx)
deploy.do_deploy(dry_run=False)
rp = helper_functions.calculate_rule_priority(ctx, ctx.stack_name)
self.assertEqual(rp, "10",
"Failed to detect existing rule priority")
rp = helper_functions.calculate_rule_priority(ctx, "foo-bar")
self.assertEqual(rp, "31", "Failed to calculate rule priority")
name = helper_functions.load_balancer_name(ctx)
print("Name: " + name)
self.assertEqual(name.count('/'), 2, "Failed to trim")
self.assertEqual(name.count(':'), 0, "Failed to trim")
self.assertTrue(name.startswith('app'), "Failed to trim")
finally:
self.clean(cloudformation=cloudformation)
def load_balancer_name(deploy_ctx):
# type: (DeployContext) -> str
cf = CloudFormationBuddy(deploy_ctx)
# we need an export value from the cluster stack so manually override it
val = _get_cluster_stack_export_value(cf, deploy_ctx,
"ElasticLoadBalancerARN")
# //#amazon conveniently wants some substring of the ARN instead of the name or other value actually available in the API
# //# turn arn:aws:elasticloadbalancing:us-west-2:271083817914:listener/app/prod-EcsEl-1WYNMMT2MT9NR/c5f92ddeb151227f/313bb2e23d9dd8d8
# //# into app/prod-EcsEl-1WYNMMT2MT9NR/c5f92ddeb151227f/313bb2e23d9dd8d8
return "" if not val else val[val.find('app/'):]
class S3Deploy(Deploy):
def __init__(self, artifact_id, location, ctx):
super(S3Deploy, self).__init__(ctx)
self.location = location
self.artifact_id = artifact_id
self.cloud_formation_buddy = CloudFormationBuddy(self.deploy_ctx)
def _internal_deploy(self, dry_run):
mkdtemp = tempfile.mkdtemp()
if not self.artifact_id.endswith(".zip"):
self.artifact_id = "{}.zip".format(self.artifact_id)
artifact_download = "s3://{location}/{artifact_id}".format(
location=self.location, artifact_id=self.artifact_id)
destination_bucket = self.cloud_formation_buddy.get_export_value(
param="WWW-Files")
s3util.download_zip_from_s3_url(artifact_download, destination=mkdtemp)
to_upload = self.get_filepaths(mkdtemp)
if dry_run:
print_utility.banner_warn(
"Dry Run: Uploading files to - {}".format(destination_bucket),
str(to_upload))
else:
split = destination_bucket.split("/")
if len(split) > 1:
path = "/".join(split[1:])
else:
path = ''
s3 = S3Buddy(self.deploy_ctx, path, split[0])
print_utility.progress("S3 Deploy: Uploading files to - {}".format(
destination_bucket))
for s3_key, path in to_upload.items():
print_utility.info("{} - {}".format(destination_bucket,
s3_key))
s3.upload(key_name=s3_key, file=path)
def get_filepaths(self, local_directory):
rel_paths = {}
for root, dirs, files in os.walk(local_directory):
for filename in files:
# construct the full local path
local_path = os.path.join(root, filename)
# construct the full Dropbox path
relative_path = os.path.relpath(local_path, local_directory)
# s3_path = os.path.join(destination, relative_path)
rel_paths[relative_path] = local_path
return rel_paths
def __str__(self):
return "{} - {}:{}".format(self.__class__.__name__, self.location,
self.artifact_id)
def calculate_rule_priority(deploy_ctx, stack_name):
# type: (DeployContext,str) -> str
cf = CloudFormationBuddy(deploy_ctx)
# we need some data for the passed stack_name so manually override it
cf.stack_name = stack_name
if cf.does_stack_exist():
return cf.get_existing_parameter_value('RulePriority')
else:
listenerArn = _get_cluster_stack_export_value(cf, deploy_ctx,
"HTTPSListenerARN")
if listenerArn:
client = get_boto_client(deploy_ctx)
rules = client.describe_rules(ListenerArn=listenerArn)['Rules']
else:
rules = None
if not rules or len(rules) == 0:
current_max = 30
else:
if len(rules) == 1 and rules[0]['Priority'] == "default":
current_max = 30
else:
current_max = int(_get_max_priority(rules))
return str(current_max + 1)
def __init__(self, deploy_ctx, run_task: bool = False):
super(ECSBuddy, self).__init__()
self.deploy_ctx = deploy_ctx
self.client = boto3.client('ecs', region_name=self.deploy_ctx.region)
self.cf = CloudFormationBuddy(deploy_ctx)
self.cluster = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.cluster_stack_name}-ECSCluster")
self.run_task = run_task
if run_task:
self.networkConfiguration = self._build_network_configuration()
else:
self.ecs_service = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSService")
self.ecs_task_family = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSTaskFamily")
self.ecs_task_execution_role = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSTaskExecutionRole")
self.ecs_task_role = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSTaskRole")
self.using_fargate = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-Fargate")
self.task_definition_description = None
self.new_image = None
def _internal_deploy(self, dry_run):
# Initialize our buddies
s3 = CloudFormationDeployS3Buddy(self.deploy_ctx)
cloud_formation = CloudFormationBuddy(self.deploy_ctx)
if dry_run:
self.validate()
return
# Upload our template to s3 to make things a bit easier and keep a record
template_file_url = s3.upload(file=(self.template_file))
# Upload all of our config files to S3 rendering any variables
config_files = self.get_rendered_config_files()
for rendered in config_files:
s3.upload(file=rendered)
# render our parameter files
parameter_file_rendered = self.get_rendered_param_file()
# see if we are updating or creating
if cloud_formation.should_create_change_set():
cloud_formation.create_change_set(
template_file_url=template_file_url,
parameter_file=parameter_file_rendered)
# make sure it is available and that there are no special conditions
if cloud_formation.should_execute_change_set():
print_utility.progress(
"Updating existing stack with ChangeSet - {}".format(
self.stack_name))
cloud_formation.execute_change_set()
else:
print_utility.warn("No computed changes for stack - {}".format(
self.stack_name))
# if there are no changes then clean up and exit
cloud_formation.delete_change_set()
return
else:
print_utility.progress("Creating new stack - {}".format(
self.stack_name))
cloud_formation.create_stack(
template_file_url=template_file_url,
parameter_file=parameter_file_rendered)
def __init__(self, artifact_id, location, ctx):
super(S3Deploy, self).__init__(ctx)
self.location = location
self.artifact_id = artifact_id
self.cloud_formation_buddy = CloudFormationBuddy(self.deploy_ctx)
def test_cloudformation_create_and_update(self):
cloudformation = CloudFormationBuddy(self.test_deploy_ctx)
s3 = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
temp_dir = tempfile.mkdtemp()
try:
template = ParentTestCase._get_resource_path("cloudformation/aws-resources.template")
parameter_file = ParentTestCase._get_resource_path("cloudformation/aws-resources.parameters.json")
self.test_deploy_ctx['RANDOM'] = self.randomWord(5)
parameter_file_rendered = self.test_deploy_ctx.render_template(parameter_file,temp_dir)
template_file_url = s3.upload(file=template)
self.assertFalse(cloudformation.does_stack_exist(), "Failed to identify reality")
cloudformation.create_stack(template_file_url=template_file_url,
parameter_file=parameter_file_rendered)
self.assertTrue(cloudformation.does_stack_exist(), "Failed to identify reality")
self.assertEqual(cloudformation.get_stack_status(), "CREATE_COMPLETE", "Failed to identify create complete")
cloudformation.create_change_set(template_file_url=template_file_url,
parameter_file=parameter_file_rendered)
self.assertFalse(cloudformation.should_execute_change_set(),"Failed to id noop changeset")
cloudformation.delete_change_set()
self.test_deploy_ctx['RANDOM'] = self.randomWord(5)
parameter_file_rendered = self.test_deploy_ctx.render_template(parameter_file,temp_dir)
cloudformation.create_change_set(template_file_url=template_file_url,
parameter_file=parameter_file_rendered)
self.assertEqual(cloudformation.get_change_set_status(refresh=True), "CREATE_COMPLETE",
"Did not get expected cs status")
self.assertTrue(cloudformation.should_execute_change_set(),"Failed to id good changeset")
if not cloudformation.should_execute_change_set():
self.fail("Did not want to execute changeset")
else:
cloudformation.execute_change_set()
cloudformation.log_stack_status()
finally:
super(CloudFormationTestCase, self).clean(cloudformation)
super(CloudFormationTestCase, self).clean_s3(s3)
super(CloudFormationTestCase, self).clean_dir(temp_dir)
class ECSBuddy(object):
def __init__(self, deploy_ctx, run_task: bool = False):
super(ECSBuddy, self).__init__()
self.deploy_ctx = deploy_ctx
self.client = boto3.client('ecs', region_name=self.deploy_ctx.region)
self.cf = CloudFormationBuddy(deploy_ctx)
self.cluster = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.cluster_stack_name}-ECSCluster")
self.run_task = run_task
if run_task:
self.networkConfiguration = self._build_network_configuration()
else:
self.ecs_service = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSService")
self.ecs_task_family = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSTaskFamily")
self.ecs_task_execution_role = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSTaskExecutionRole")
self.ecs_task_role = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-ECSTaskRole")
self.using_fargate = self.cf.wait_for_export(
fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-Fargate")
self.task_definition_description = None
self.new_image = None
def set_container_image(self, location, tag):
self.new_image = f"{location}:{tag}"
def requires_update(self):
if not self.new_image:
print_utility.warn("Checking for ECS update without registering new image ")
return False
if not self.ecs_task_family:
print_utility.warn("No ECS Task family found - assuming first deploy of stack and skipping ECS update")
return False
self._describe_task_definition()
existing = pydash.get(self.task_definition_description, "containerDefinitions[0].image")
print_utility.info(f"ECS task existing image - {existing}")
print_utility.info(f"ECS task desired image - {self.new_image}")
return self.run_task or existing != self.new_image
def perform_update(self):
self._describe_task_definition(refresh=True)
new_task_def = {
'family': self.task_definition_description['family'],
'containerDefinitions': self.task_definition_description['containerDefinitions'],
'volumes': self.task_definition_description['volumes']
}
if 'networkMode' in self.task_definition_description:
new_task_def['networkMode'] = self.task_definition_description['networkMode']
new_task_def['containerDefinitions'][0]['image'] = self.new_image
ctx_memory = self.deploy_ctx.get('TASK_MEMORY')
if ctx_memory:
new_task_def['containerDefinitions'][0]['memory'] = ctx_memory
if 'TASK_SOFT_MEMORY' in self.deploy_ctx and self.deploy_ctx['TASK_SOFT_MEMORY']:
new_task_def['containerDefinitions'][0]['memoryReservation'] = self.deploy_ctx['TASK_SOFT_MEMORY']
ctx_cpu = self.deploy_ctx.get('TASK_CPU')
if ctx_cpu:
new_task_def['containerDefinitions'][0]['cpu'] = ctx_cpu
# set at the task level for fargate definitions
if self.using_fargate:
first_container = new_task_def['containerDefinitions'][0]
new_task_def['requiresCompatibilities'] = ['FARGATE']
new_cpu = ctx_cpu or first_container.get('cpu')
if new_cpu:
new_task_def['cpu'] = str(new_cpu) # not sure if this is right but AWS says it should be str
new_memory = ctx_memory or first_container.get('memoryReservation')
if new_memory:
new_task_def['memory'] = str(new_memory) # not sure if this is right but AWS says it should be str
if self.ecs_task_execution_role:
new_task_def['executionRoleArn'] = self.ecs_task_execution_role
if self.ecs_task_role:
new_task_def['taskRoleArn'] = self.ecs_task_role
for k, v in self.deploy_ctx.items():
print_utility.info(f'[deploy_ctx] {k} = {repr(v)}')
for k, v in new_task_def.items():
print_utility.info(f'[new_task_def] {k} = {repr(v)}')
updated_task_definition = self.client.register_task_definition(**new_task_def)['taskDefinition']
new_task_def_arn = updated_task_definition['taskDefinitionArn']
if self.run_task:
self.exec_run_task(new_task_def_arn)
else:
self.update_service(new_task_def_arn)
def update_service(self, new_task_def_arn):
self.deploy_ctx.notify_event(
title=f"Update of ecs service {self.ecs_service} started",
type="success")
self.client.update_service(
cluster=self.cluster,
service=self.ecs_service,
taskDefinition=new_task_def_arn)
waiter = self.client.get_waiter('services_stable')
success = True
try:
waiter.wait(
cluster=self.cluster,
services=[self.ecs_service]
)
except WaiterError as e:
success = False
print_utility.error(f"Error waiting for service to stabilize - {e}", raise_exception=True)
finally:
self.deploy_ctx.notify_event(
title=f"Update of ecs service {self.ecs_service} completed: {'Success' if success else 'Failed'}",
type="success" if success else "error")
def _describe_task_definition(self, refresh=False):
if self.task_definition_description and not refresh:
return
self.task_definition_description = self.client.describe_task_definition(taskDefinition=
self.ecs_task_family)['taskDefinition']
def exec_run_task(self, new_task_def_arn):
self.deploy_ctx.notify_event(
title=f"Running one time ecs task with image: {self.new_image}",
type="success")
ret = self.client.run_task(cluster=self.cluster,
launchType='FARGATE' if self.using_fargate else 'EC2',
taskDefinition=new_task_def_arn,
networkConfiguration=self.networkConfiguration)
success = True
try:
if self.deploy_ctx.wait_for_run_task_finish():
waiter_name = 'tasks_stopped'
else:
waiter_name = 'tasks_running'
waiter = self.client.get_waiter(waiter_name=waiter_name)
waiter.wait(cluster=self.cluster, tasks=[ret['tasks'][0]['taskArn']])
except WaiterError as e:
success = False
print_utility.error(f"Error waiting for task to run - {e}", raise_exception=True)
finally:
self.deploy_ctx.notify_event(
title=f"Task running with started: {'Success' if success else 'Failed'}: Image - {self.new_image} ",
type="success" if success else "error")
def _build_network_configuration(self):
# {'awsvpcConfiguration': {'subnets': ['subnet-030d5ae3a5aed3c30', 'subnet-0058692e292b68211', 'subnet-02ca3729edae57a6a'],
# 'securityGroups': ['sg-087a1bbf3463e7614'], 'assignPublicIp': 'DISABLED'}}
subnets = []
for sub in ['A', 'B', 'C']:
subnet = self.cf.wait_for_export(fully_qualified_param_name=f"{self.deploy_ctx.vpc_stack_name}-Subnet{sub}Priv")
subnets.append(subnet)
sec_group = self.cf.wait_for_export(fully_qualified_param_name=f"{self.deploy_ctx.stack_name}-FargateSecurityGroup")
return {
'awsvpcConfiguration': {'subnets': subnets, 'securityGroups': [sec_group], 'assignPublicIp': 'DISABLED'}}
def what_is_your_plan(self):
if self.run_task:
return f"ECS Deploy running task using image {self.new_image}"
else:
return f"ECS Deploy updating service {self.ecs_service} to use image {self.new_image}"