def get_existing_parameter_value(self, param_val):
self._describe_stack()
for param in self.stack_description.get('Parameters', []):
if param['ParameterKey'] == param_val:
return param['ParameterValue']
print_utility.error("Could not locate parameter value: {}".format(param_val))
return None
def __init__(self, artifact_directory, environment):
super(ServiceDefinition, self).__init__()
self.artifact_directory = artifact_directory
service_definition_path = os.path.join(artifact_directory,
_SERVICE_DEFINITION_FILE)
if not os.path.exists(service_definition_path):
err_msg = "Service definition ({}) does not exist in artifact directory - {}".format(
_SERVICE_DEFINITION_FILE, artifact_directory)
print_utility.error(err_msg)
raise Exception(err_msg)
with open(service_definition_path, 'r') as fp:
service_definition = json.load(fp)
validate(service_definition, self.schema)
self.application = service_definition[_APPLICATION]
self.role = service_definition[_ROLE]
self.service_type = service_definition[_SERVICE_TYPE]
self.docker_registry = service_definition.get(_DOCKER_REGISTRY, "")
self.service_template_definition_locations = service_definition.get(
_SERVICE_TEMPLATE_DEFINITION_LOCATIONS, [])
if _DEPLOYMENT_PARAMETERS in service_definition:
self.deployment_parameters = service_definition[
_DEPLOYMENT_PARAMETERS]
env_deployment_parameters = '{environment}-deployment-parameters'.format(
environment=environment)
if env_deployment_parameters in service_definition:
print_utility.info(
"Updating deployment params with environment"
" specific settings - {}".format(
env_deployment_parameters))
self.deployment_parameters.update(
service_definition[env_deployment_parameters])
print_utility.info("Loaded deployment parameters: " +
pformat(self.deployment_parameters, indent=4))
self.service_modifications = service_definition.get(
_MODIFICATIONS, [])
def _validate_template_dir(self, err_on_failure_to_locate=True):
if not os.path.exists(self.get_defaults_file_path()):
if err_on_failure_to_locate:
print_utility.error(
"Remote Defaults file could not be "
"located for service - {service_type}".format(
service_type=self.service_type),
raise_exception=True)
def init_dd(self):
api_key = self.deploy_ctx.get('DATADOG_KEY', None)
app_key = self.deploy_ctx.get('DATADOG_APP_KEY', None)
if api_key is None or app_key is None:
print_utility.error(
"Can not deploy datadog monitor without configuring DATADOG_KEY and DATADOG_APP_KEY",
raise_exception=True)
dd.initialize(api_key=api_key, app_key=app_key)
def should_create_change_set(self):
exists = self.does_stack_exist()
if exists:
if self.get_stack_status() == 'ROLLBACK_COMPLETE':
print_utility.error("Can not update stack in state 'ROLLBACK_COMPLETE' -"
" delete stack to recreate.",
raise_exception=True)
return exists
def locate_service(self, service_type):
# type: (str, bool) -> Template
template = self.deploy_templates.get(service_type, None)
if not template:
print_utility.error(
"Unknown service template - {}".format(service_type),
raise_exception=True)
template.download_template()
return template
def transform(self, definition, value):
func_name = definition['func_name']
if 'transform_fargate_cpu' == func_name:
return helper_functions.transform_fargate_cpu(self.defaults, value)
elif 'transform_fargate_memory' == func_name:
return helper_functions.transform_fargate_memory(
self.defaults, value)
else:
print_utility.error(
"Can not locate function for defaults.json: Stack {} Function {}"
.format(self.stack_name, func_name))
def download_url_to_destination(url, destination):
r = requests.get(url, stream=True)
if r.status_code != 200:
print_utility.error(
"Template could not be downloaded - {url} {status} {body}".
format(url=url, status=r.status_code, body=r.text))
temporary_file = tempfile.NamedTemporaryFile()
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
temporary_file.write(chunk)
temporary_file.seek(0)
with ZipFile(temporary_file) as zf:
zf.extractall(destination)
def locate_service_modification(self, service_type, mod_type):
# type: (str, str) -> Template
template = self.get_service_modifications_for_service(
service_type).get(mod_type, None)
if not template:
print_utility.error(
"Unknown service modification '{}' for type '{}'"
" Known modifications are {}".format(
mod_type, service_type,
self.get_service_modifications_for_service(
service_type=service_type)),
raise_exception=True)
template.download_template()
return template
def transform_fargate_memory(deploy_ctx, _value):
if _using_fargate(deploy_ctx):
if isinstance(_value, str):
if _value not in _valid_fargate_memories:
print_utility.error(
'Attempting to use fargate with invalid memory. {} Memory Valid Values: {}'
.format(_value, _valid_fargate_memories),
raise_exception=True)
else:
memory = _value
else:
memory = "{}".format(_get_valid_fargate_memory(_value))
_validate_fargate_resource_allocation(None, memory, deploy_ctx)
return memory
def get_known_template(self, template_name):
template = self.deploy_templates.get(
template_name,
self.default_service_modification_templates.get(template_name))
if not template:
template = self.all_service_mods.get(template_name)
if template:
template.download_template()
return template
else:
print_utility.error(
f"Unknown service template - {template_name} "
f"- known templates are deploy_templates={self.deploy_templates.keys()} "
f"- service_mod_templates={self.all_service_mods.keys()}",
raise_exception=True)
def _validate_template_dir(self, err_on_failure_to_locate=True):
if not os.path.exists(self.get_template_file_path()):
if err_on_failure_to_locate:
print_utility.error(
f"Template file could not be located for service - "
f"{self.service_type} - {self.__str__()}",
raise_exception=True)
return
if not os.path.exists(self.get_parameter_file_path()):
if err_on_failure_to_locate:
print_utility.error(
f"Parameter file could not be located for service - "
f"{self.service_type} - {self.__str__()}",
raise_exception=True)
return
self.valid = True
def _load_value(self, value, key, transformations):
type_ = value['type']
if type_ == _PARAM_TYPE_TEMPLATE:
return self.deploy_ctx.expandvars(value['value'], self.defaults)
elif type_ == _PARAM_TYPE_FUNC:
if 'default_key' in value:
# look for a default value before calling the func
def_key = value['default_key']
if def_key in self.defaults:
return self.defaults[def_key]
elif def_key in self.deploy_ctx:
return self.deploy_ctx[def_key]
# someday make it dynamic
func_name = value['func_name']
if "load_balancer_name" == func_name:
return helper_functions.load_balancer_name(self.deploy_ctx)
elif 'rule_priority' == func_name:
return helper_functions.calculate_rule_priority(
self.deploy_ctx, self.stack_name)
else:
print_utility.error(
"Can not locate function for defaults.json: Stack {} Function {}"
.format(self.stack_name, func_name))
elif type_ == _PARAM_TYPE_PROPERTY:
default_value = value.get('default', None)
if isinstance(default_value, str):
default_value = self.deploy_ctx.expandvars(
str(default_value), self.defaults)
return self.deploy_ctx.get(value['key'], default_value)
elif type_ == _PARAM_TYPE_TRANSFORM:
# add it to the list of properties to transform after load
transformations[key] = value
# Load like a normal property, so override the type
value['type'] = _PARAM_TYPE_PROPERTY
# and recurse
return self._load_value(value, None, None)
else:
# should die on JSON validation but to be complete
print_utility.error(
"Can not load value for type in defaults.json: Stack {} Type {}"
.format(self.stack_name, type_))
def _load_templates(self, templates, service_modification=False):
# type: (dict, bool) -> None
validate(templates, self.schema)
alias_templates = []
for name, values in templates.items():
type_ = values['type']
if type_ == "github":
template = GitHubTemplate(service_type=name, values=values)
elif type_ == "s3":
template = S3Template(service_type=name, values=values)
elif type_ == "url":
template = URLTemplate(service_type=name, values=values)
elif type_ == "alias":
template = AliasTemplate(service_type=name, values=values)
alias_templates.append(template)
else:
print_utility.error(
"Can not locate resource. Requested unknown template type - {}"
.format(type_),
raise_exception=True)
raise Exception("")
if service_modification:
compatibility = values.get('compatible', [])
for service in compatibility:
if service == "*":
self.default_service_modification_templates[
name] = template
else:
self.service_modification_templates[service][
name] = template
self.all_service_mods[name] = template
else:
if name in self.deploy_templates:
print_utility.info(
f"Overwriting existing template for service {name}: {self.deploy_templates[name]}"
)
self.deploy_templates[name] = template
for alias in alias_templates:
alias.resolve(self.all_service_mods
if service_modification else self.deploy_templates)
def _internal_deploy(self, dry_run):
to_deploy = self.expand_monitors()
for monitor in to_deploy:
print_utility.info("Deploying datadog monitor: {}".format(
monitor['name']))
if not dry_run:
self.init_dd()
existing_id = self.find_monitor_if_exists(monitor['name'])
if not existing_id:
response = dd.api.Monitor.create(**monitor)
created_name = response.get('name', None)
if created_name:
print_utility.info(
"Created monitor - {}".format(created_name))
else:
print_utility.error(
"Error creating monitor - {}".format(response),
raise_exception=True)
else:
response = dd.api.Monitor.update(id=existing_id, **monitor)
print_utility.info("Updated monitor - {}".format(
response['name']))
def update_service(self, new_task_def_arn):
self.deploy_ctx.notify_event(
title=f"Update of ecs service {self.ecs_service} started",
type="success")
self.client.update_service(
cluster=self.cluster,
service=self.ecs_service,
taskDefinition=new_task_def_arn)
waiter = self.client.get_waiter('services_stable')
success = True
try:
waiter.wait(
cluster=self.cluster,
services=[self.ecs_service]
)
except WaiterError as e:
success = False
print_utility.error(f"Error waiting for service to stabilize - {e}", raise_exception=True)
finally:
self.deploy_ctx.notify_event(
title=f"Update of ecs service {self.ecs_service} completed: {'Success' if success else 'Failed'}",
type="success" if success else "error")
def exec_run_task(self, new_task_def_arn):
self.deploy_ctx.notify_event(
title=f"Running one time ecs task with image: {self.new_image}",
type="success")
ret = self.client.run_task(cluster=self.cluster,
launchType='FARGATE' if self.using_fargate else 'EC2',
taskDefinition=new_task_def_arn,
networkConfiguration=self.networkConfiguration)
success = True
try:
if self.deploy_ctx.wait_for_run_task_finish():
waiter_name = 'tasks_stopped'
else:
waiter_name = 'tasks_running'
waiter = self.client.get_waiter(waiter_name=waiter_name)
waiter.wait(cluster=self.cluster, tasks=[ret['tasks'][0]['taskArn']])
except WaiterError as e:
success = False
print_utility.error(f"Error waiting for task to run - {e}", raise_exception=True)
finally:
self.deploy_ctx.notify_event(
title=f"Task running with started: {'Success' if success else 'Failed'}: Image - {self.new_image} ",
type="success" if success else "error")
def _validate_fargate_resource_allocation(cpu, memory, deploy_ctx):
if cpu is None:
discovered_cpu = deploy_ctx.get('TASK_CPU', None)
if discovered_cpu not in _valid_fargate_resources:
print_utility.info(
'Skipping fargate resource validation - CPU not transformed - {}'
.format(discovered_cpu))
return
cpu = discovered_cpu
elif memory is None:
discovered_memory = deploy_ctx.get('TASK_SOFT_MEMORY', None)
if discovered_memory not in _valid_fargate_memories:
print_utility.info(
'Skipping fargate resource validation - Memory not transformed - {}'
.format(discovered_memory))
return
memory = discovered_memory
memory_possibilities = _valid_fargate_resources[cpu]
if memory not in memory_possibilities:
print_utility.error(
'Attempting to use fargate with invalid configuration. {} CPU {} Memory'
.format(cpu, memory),
raise_exception=True)
def do_command(deploy_ctx, service_template_directory=None, service_type=None):
# type: (DeployContext,[str or None],str) -> None
if service_template_directory is None:
print_utility.info(
"Service template directory was not provided. Assuming service-type '{}' is built-in."
.format(service_type))
template = deploy_ctx.template_manager.get_known_template(
template_name=service_type)
deploy = CloudFormationDeploy(stack_name=deploy_ctx.stack_name,
template=template,
deploy_ctx=deploy_ctx)
else:
deploy = CloudFormationDeploy(
stack_name=deploy_ctx.stack_name,
template=NamedLocalTemplate(service_template_directory),
deploy_ctx=deploy_ctx)
errs = deploy.analyze()
if errs > 0:
print_utility.error("Template raised {} errors ".format(errs),
raise_exception=True)
else:
print_utility.banner_warn(
"Service Template Validation - {}".format(service_type),
"SUCCESS - No errors")
def _print_error(self, errors):
for key, errs in errors.items():
print_utility.error(pformat(key, indent=4))
print_utility.banner(pformat(errs, indent=8))