Пример #1
0
 def get_existing_parameter_value(self, param_val):
     self._describe_stack()
     for param in self.stack_description.get('Parameters', []):
         if param['ParameterKey'] == param_val:
             return param['ParameterValue']
     print_utility.error("Could not locate parameter value: {}".format(param_val))
     return None
Пример #2
0
 def __init__(self, artifact_directory, environment):
     super(ServiceDefinition, self).__init__()
     self.artifact_directory = artifact_directory
     service_definition_path = os.path.join(artifact_directory,
                                            _SERVICE_DEFINITION_FILE)
     if not os.path.exists(service_definition_path):
         err_msg = "Service definition ({}) does not exist in artifact directory - {}".format(
             _SERVICE_DEFINITION_FILE, artifact_directory)
         print_utility.error(err_msg)
         raise Exception(err_msg)
     with open(service_definition_path, 'r') as fp:
         service_definition = json.load(fp)
         validate(service_definition, self.schema)
         self.application = service_definition[_APPLICATION]
         self.role = service_definition[_ROLE]
         self.service_type = service_definition[_SERVICE_TYPE]
         self.docker_registry = service_definition.get(_DOCKER_REGISTRY, "")
         self.service_template_definition_locations = service_definition.get(
             _SERVICE_TEMPLATE_DEFINITION_LOCATIONS, [])
         if _DEPLOYMENT_PARAMETERS in service_definition:
             self.deployment_parameters = service_definition[
                 _DEPLOYMENT_PARAMETERS]
         env_deployment_parameters = '{environment}-deployment-parameters'.format(
             environment=environment)
         if env_deployment_parameters in service_definition:
             print_utility.info(
                 "Updating deployment params with environment"
                 " specific settings - {}".format(
                     env_deployment_parameters))
             self.deployment_parameters.update(
                 service_definition[env_deployment_parameters])
         print_utility.info("Loaded deployment parameters: " +
                            pformat(self.deployment_parameters, indent=4))
         self.service_modifications = service_definition.get(
             _MODIFICATIONS, [])
Пример #3
0
 def _validate_template_dir(self, err_on_failure_to_locate=True):
     if not os.path.exists(self.get_defaults_file_path()):
         if err_on_failure_to_locate:
             print_utility.error(
                 "Remote Defaults file could not be "
                 "located for service - {service_type}".format(
                     service_type=self.service_type),
                 raise_exception=True)
 def init_dd(self):
     api_key = self.deploy_ctx.get('DATADOG_KEY', None)
     app_key = self.deploy_ctx.get('DATADOG_APP_KEY', None)
     if api_key is None or app_key is None:
         print_utility.error(
             "Can not deploy datadog monitor without configuring DATADOG_KEY and DATADOG_APP_KEY",
             raise_exception=True)
     dd.initialize(api_key=api_key, app_key=app_key)
Пример #5
0
 def should_create_change_set(self):
     exists = self.does_stack_exist()
     if exists:
         if self.get_stack_status() == 'ROLLBACK_COMPLETE':
             print_utility.error("Can not update stack in state 'ROLLBACK_COMPLETE' -"
                                 " delete stack to recreate.",
                                 raise_exception=True)
     return exists
Пример #6
0
 def locate_service(self, service_type):
     # type: (str, bool) -> Template
     template = self.deploy_templates.get(service_type, None)
     if not template:
         print_utility.error(
             "Unknown service template - {}".format(service_type),
             raise_exception=True)
     template.download_template()
     return template
Пример #7
0
 def transform(self, definition, value):
     func_name = definition['func_name']
     if 'transform_fargate_cpu' == func_name:
         return helper_functions.transform_fargate_cpu(self.defaults, value)
     elif 'transform_fargate_memory' == func_name:
         return helper_functions.transform_fargate_memory(
             self.defaults, value)
     else:
         print_utility.error(
             "Can not locate function for defaults.json: Stack {} Function {}"
             .format(self.stack_name, func_name))
Пример #8
0
 def download_url_to_destination(url, destination):
     r = requests.get(url, stream=True)
     if r.status_code != 200:
         print_utility.error(
             "Template could not be downloaded - {url} {status} {body}".
             format(url=url, status=r.status_code, body=r.text))
     temporary_file = tempfile.NamedTemporaryFile()
     for chunk in r.iter_content(chunk_size=1024):
         if chunk:  # filter out keep-alive new chunks
             temporary_file.write(chunk)
     temporary_file.seek(0)
     with ZipFile(temporary_file) as zf:
         zf.extractall(destination)
Пример #9
0
 def locate_service_modification(self, service_type, mod_type):
     # type: (str, str) -> Template
     template = self.get_service_modifications_for_service(
         service_type).get(mod_type, None)
     if not template:
         print_utility.error(
             "Unknown service modification '{}' for type '{}'"
             " Known modifications are {}".format(
                 mod_type, service_type,
                 self.get_service_modifications_for_service(
                     service_type=service_type)),
             raise_exception=True)
     template.download_template()
     return template
Пример #10
0
def transform_fargate_memory(deploy_ctx, _value):
    if _using_fargate(deploy_ctx):
        if isinstance(_value, str):
            if _value not in _valid_fargate_memories:
                print_utility.error(
                    'Attempting to use fargate with invalid memory.  {} Memory Valid Values: {}'
                    .format(_value, _valid_fargate_memories),
                    raise_exception=True)
            else:
                memory = _value
        else:
            memory = "{}".format(_get_valid_fargate_memory(_value))
        _validate_fargate_resource_allocation(None, memory, deploy_ctx)
        return memory
Пример #11
0
 def get_known_template(self, template_name):
     template = self.deploy_templates.get(
         template_name,
         self.default_service_modification_templates.get(template_name))
     if not template:
         template = self.all_service_mods.get(template_name)
     if template:
         template.download_template()
         return template
     else:
         print_utility.error(
             f"Unknown service template - {template_name} "
             f"- known templates are deploy_templates={self.deploy_templates.keys()} "
             f"- service_mod_templates={self.all_service_mods.keys()}",
             raise_exception=True)
Пример #12
0
 def _validate_template_dir(self, err_on_failure_to_locate=True):
     if not os.path.exists(self.get_template_file_path()):
         if err_on_failure_to_locate:
             print_utility.error(
                 f"Template file could not be located for service - "
                 f"{self.service_type} - {self.__str__()}",
                 raise_exception=True)
         return
     if not os.path.exists(self.get_parameter_file_path()):
         if err_on_failure_to_locate:
             print_utility.error(
                 f"Parameter file could not be located for service  - "
                 f"{self.service_type} - {self.__str__()}",
                 raise_exception=True)
         return
     self.valid = True
Пример #13
0
 def _load_value(self, value, key, transformations):
     type_ = value['type']
     if type_ == _PARAM_TYPE_TEMPLATE:
         return self.deploy_ctx.expandvars(value['value'], self.defaults)
     elif type_ == _PARAM_TYPE_FUNC:
         if 'default_key' in value:
             # look for a default value before calling the func
             def_key = value['default_key']
             if def_key in self.defaults:
                 return self.defaults[def_key]
             elif def_key in self.deploy_ctx:
                 return self.deploy_ctx[def_key]
         # someday make it dynamic
         func_name = value['func_name']
         if "load_balancer_name" == func_name:
             return helper_functions.load_balancer_name(self.deploy_ctx)
         elif 'rule_priority' == func_name:
             return helper_functions.calculate_rule_priority(
                 self.deploy_ctx, self.stack_name)
         else:
             print_utility.error(
                 "Can not locate function for defaults.json: Stack {} Function {}"
                 .format(self.stack_name, func_name))
     elif type_ == _PARAM_TYPE_PROPERTY:
         default_value = value.get('default', None)
         if isinstance(default_value, str):
             default_value = self.deploy_ctx.expandvars(
                 str(default_value), self.defaults)
         return self.deploy_ctx.get(value['key'], default_value)
     elif type_ == _PARAM_TYPE_TRANSFORM:
         # add it to the list of properties to transform after load
         transformations[key] = value
         # Load like a normal property, so override the type
         value['type'] = _PARAM_TYPE_PROPERTY
         # and recurse
         return self._load_value(value, None, None)
     else:
         # should die on JSON validation but to be complete
         print_utility.error(
             "Can not load value for type in defaults.json: Stack {} Type {}"
             .format(self.stack_name, type_))
Пример #14
0
    def _load_templates(self, templates, service_modification=False):
        # type: (dict, bool) -> None
        validate(templates, self.schema)
        alias_templates = []
        for name, values in templates.items():
            type_ = values['type']
            if type_ == "github":
                template = GitHubTemplate(service_type=name, values=values)
            elif type_ == "s3":
                template = S3Template(service_type=name, values=values)
            elif type_ == "url":
                template = URLTemplate(service_type=name, values=values)
            elif type_ == "alias":
                template = AliasTemplate(service_type=name, values=values)
                alias_templates.append(template)
            else:
                print_utility.error(
                    "Can not locate resource. Requested unknown template type - {}"
                    .format(type_),
                    raise_exception=True)
                raise Exception("")

            if service_modification:
                compatibility = values.get('compatible', [])
                for service in compatibility:
                    if service == "*":
                        self.default_service_modification_templates[
                            name] = template
                    else:
                        self.service_modification_templates[service][
                            name] = template
                    self.all_service_mods[name] = template
            else:
                if name in self.deploy_templates:
                    print_utility.info(
                        f"Overwriting existing template for service {name}: {self.deploy_templates[name]}"
                    )
                self.deploy_templates[name] = template
        for alias in alias_templates:
            alias.resolve(self.all_service_mods
                          if service_modification else self.deploy_templates)
Пример #15
0
 def _internal_deploy(self, dry_run):
     to_deploy = self.expand_monitors()
     for monitor in to_deploy:
         print_utility.info("Deploying datadog monitor: {}".format(
             monitor['name']))
         if not dry_run:
             self.init_dd()
             existing_id = self.find_monitor_if_exists(monitor['name'])
             if not existing_id:
                 response = dd.api.Monitor.create(**monitor)
                 created_name = response.get('name', None)
                 if created_name:
                     print_utility.info(
                         "Created monitor - {}".format(created_name))
                 else:
                     print_utility.error(
                         "Error creating monitor - {}".format(response),
                         raise_exception=True)
             else:
                 response = dd.api.Monitor.update(id=existing_id, **monitor)
                 print_utility.info("Updated monitor - {}".format(
                     response['name']))
Пример #16
0
 def update_service(self, new_task_def_arn):
     self.deploy_ctx.notify_event(
         title=f"Update of ecs service {self.ecs_service} started",
         type="success")
     self.client.update_service(
         cluster=self.cluster,
         service=self.ecs_service,
         taskDefinition=new_task_def_arn)
     waiter = self.client.get_waiter('services_stable')
     success = True
     try:
         waiter.wait(
             cluster=self.cluster,
             services=[self.ecs_service]
         )
     except WaiterError as e:
         success = False
         print_utility.error(f"Error waiting for service to stabilize - {e}", raise_exception=True)
     finally:
         self.deploy_ctx.notify_event(
             title=f"Update of ecs service {self.ecs_service} completed: {'Success' if success else 'Failed'}",
             type="success" if success else "error")
Пример #17
0
 def exec_run_task(self, new_task_def_arn):
     self.deploy_ctx.notify_event(
         title=f"Running one time ecs task with image: {self.new_image}",
         type="success")
     ret = self.client.run_task(cluster=self.cluster,
                                launchType='FARGATE' if self.using_fargate else 'EC2',
                                taskDefinition=new_task_def_arn,
                                networkConfiguration=self.networkConfiguration)
     success = True
     try:
         if self.deploy_ctx.wait_for_run_task_finish():
             waiter_name = 'tasks_stopped'
         else:
             waiter_name = 'tasks_running'
         waiter = self.client.get_waiter(waiter_name=waiter_name)
         waiter.wait(cluster=self.cluster, tasks=[ret['tasks'][0]['taskArn']])
     except WaiterError as e:
         success = False
         print_utility.error(f"Error waiting for task to run - {e}", raise_exception=True)
     finally:
         self.deploy_ctx.notify_event(
             title=f"Task running with started: {'Success' if success else 'Failed'}: Image - {self.new_image} ",
             type="success" if success else "error")
Пример #18
0
def _validate_fargate_resource_allocation(cpu, memory, deploy_ctx):
    if cpu is None:
        discovered_cpu = deploy_ctx.get('TASK_CPU', None)
        if discovered_cpu not in _valid_fargate_resources:
            print_utility.info(
                'Skipping fargate resource validation - CPU not transformed - {}'
                .format(discovered_cpu))
            return
        cpu = discovered_cpu
    elif memory is None:
        discovered_memory = deploy_ctx.get('TASK_SOFT_MEMORY', None)
        if discovered_memory not in _valid_fargate_memories:
            print_utility.info(
                'Skipping fargate resource validation - Memory not transformed - {}'
                .format(discovered_memory))
            return
        memory = discovered_memory
    memory_possibilities = _valid_fargate_resources[cpu]
    if memory not in memory_possibilities:
        print_utility.error(
            'Attempting to use fargate with invalid configuration.  {} CPU {} Memory'
            .format(cpu, memory),
            raise_exception=True)
Пример #19
0
def do_command(deploy_ctx, service_template_directory=None, service_type=None):
    # type: (DeployContext,[str or None],str) -> None
    if service_template_directory is None:
        print_utility.info(
            "Service template directory was not provided.  Assuming service-type '{}' is built-in."
            .format(service_type))
        template = deploy_ctx.template_manager.get_known_template(
            template_name=service_type)
        deploy = CloudFormationDeploy(stack_name=deploy_ctx.stack_name,
                                      template=template,
                                      deploy_ctx=deploy_ctx)
    else:
        deploy = CloudFormationDeploy(
            stack_name=deploy_ctx.stack_name,
            template=NamedLocalTemplate(service_template_directory),
            deploy_ctx=deploy_ctx)
    errs = deploy.analyze()
    if errs > 0:
        print_utility.error("Template raised {} errors ".format(errs),
                            raise_exception=True)
    else:
        print_utility.banner_warn(
            "Service Template Validation - {}".format(service_type),
            "SUCCESS - No errors")
Пример #20
0
 def _print_error(self, errors):
     for key, errs in errors.items():
         print_utility.error(pformat(key, indent=4))
         print_utility.banner(pformat(errs, indent=8))