def __init__( self, hook_config: dict, config: Config, tests: Mapping[str, TestObj], parameters: dict, outputs: Optional[dict], ): super().__init__(hook_config, config, tests, parameters, outputs) if hook_config.get("generate_failure"): raise TaskCatException("generated failure from hook")
def _assert_param_combo(self): throw = False if self._stack_name_prefix and self._stack_name_suffix: throw = True if self._stack_name and (self._stack_name_prefix or self._stack_name_suffix): throw = True if throw: raise TaskCatException( "Please provide only *ONE* of stack_name, stack_name_prefix, \ or stack_name_suffix")
def create_stacks(self, threads: int = 8): if self.stacks: raise TaskCatException( "Stacker already initialised with stack objects") tests = self._tests_to_list(self.config.tests) tags = [Tag({"Key": "taskcat-id", "Value": self.uid.hex})] tags += [ Tag(t) for t in self.tags if t.key not in ["taskcat-project-name", "taskcat-test-name", "taskcat-id"] ] fan_out(self._create_stacks_for_test, {"tags": tags}, tests, threads)
def param_list_to_dict(original_keys): # Setup a list index dictionary. # - Used to give an Parameter => Index mapping for replacement. param_index = {} if not isinstance(original_keys, list): raise TaskCatException( 'Invalid parameter file, outermost json element must be a list ("[]")' ) for (idx, param_dict) in enumerate(original_keys): if not isinstance(param_dict, dict): raise TaskCatException( 'Invalid parameter %s parameters must be of type dict ("{}")' % param_dict) if "ParameterKey" not in param_dict or "ParameterValue" not in param_dict: raise TaskCatException( f"Invalid parameter {param_dict} all items must " f"have both ParameterKey and ParameterValue keys") key = param_dict["ParameterKey"] param_index[key] = idx return param_index
def param_list_to_dict(original_keys): # Setup a list index dictionary. # - Used to give an Parameter => Index mapping for replacement. param_index = {} if type(original_keys) != list: raise TaskCatException( 'Invalid parameter file, outermost json element must be a list ("[]")' ) for (idx, param_dict) in enumerate(original_keys): if type(param_dict) != dict: raise TaskCatException( 'Invalid parameter %s parameters must be of type dict ("{}")' % param_dict) if 'ParameterKey' not in param_dict or 'ParameterValue' not in param_dict: raise TaskCatException( 'Invalid parameter %s all items must have both ParameterKey and ParameterValue keys' % param_dict) key = param_dict['ParameterKey'] param_index[key] = idx return param_index
def _get_account_info(self, profile): partition, region = self._get_partition(profile) session = self.session(profile, region) sts_client = session.client("sts", region_name=region) try: account_id = sts_client.get_caller_identity()["Account"] except ClientError as e: if e.response["Error"]["Code"] == "AccessDenied": raise TaskCatException( f"Not able to fetch account number from {region} using profile " f"{profile}. {str(e)}") raise except NoCredentialsError as e: raise TaskCatException( f"Not able to fetch account number from {region} using profile " f"{profile}. {str(e)}") except ProfileNotFound as e: raise TaskCatException( f"Not able to fetch account number from {region} using profile " f"{profile}. {str(e)}") return {"partition": partition, "account_id": account_id}
def validate_output_dir(directory): if os.path.isfile(directory): directory = os.path.split(directory)[0] if not os.path.isdir(directory): LOG.info( "Directory [{}] does not exist. Trying to create it.".format( directory)) os.makedirs(directory) elif not os.access(directory, os.W_OK): raise TaskCatException( f"No write access allowed to output directory " f"[{directory}]. Aborting.")
def _sync( # noqa: C901 self, local_list, s3_list, bucket, prefix, acl, threads=16 ): # pylint: disable=too-many-locals # determine which files to remove from S3 remove_from_s3 = [] for s3_file in s3_list.keys(): if s3_file not in local_list.keys() and not self._exclude_remote(s3_file): if self.dry_run: LOG.info( f"[DRY RUN] s3://{bucket}/{prefix + prefix + s3_file}", extra={"nametag": PrintMsg.S3DELETE}, ) else: LOG.info( f"s3://{bucket}/{prefix + prefix + s3_file}", extra={"nametag": PrintMsg.S3DELETE}, ) remove_from_s3.append({"Key": prefix + s3_file}) # deleting objects, max 1k objects per s3 delete_objects call if not self.dry_run: for objects in [ remove_from_s3[i : i + 1000] for i in range(0, len(remove_from_s3), 1000) ]: response = self.s3_client.delete_objects( Bucket=bucket, Delete={"Objects": objects} ) if "Errors" in response.keys(): for error in response["Errors"]: LOG.error("S3 delete error: %s" % str(error)) raise TaskCatException("Failed to delete one or more files from S3") # build list of files to upload upload_to_s3 = [] for local_file in local_list: upload = False # If file is not present in S3 if local_file not in s3_list.keys(): upload = True # If checksum is different elif local_list[local_file][1] != s3_list[local_file]: upload = True if upload: absolute_path = local_list[local_file][0] s3_path = local_file upload_to_s3.append([absolute_path, bucket, s3_path]) # multithread the uploading of files pool = ThreadPool(threads) func = partial( self._s3_upload_file, prefix=prefix, s3_client=self.s3_client, acl=acl ) pool.map(func, upload_to_s3) pool.close() pool.join()
def parse_legacy_config(project_root: Path): config_file = (project_root / "ci/taskcat.yml").expanduser().resolve() if not config_file.is_file(): raise TaskCatException(f"No config_file at {config_file}") with open(str(config_file), "r") as file_handle: config_dict = yaml.safe_load(file_handle) # need to rename global key, as it's a python keyword config_dict["global_"] = config_dict.pop("global") legacy_config = LegacyConfig.from_dict(config_dict) tests = {} for test_name, test_data in legacy_config.tests.items(): parameters = {} parameter_file = project_root / "ci/" / test_data.parameter_input parameter_file = parameter_file.expanduser().resolve() with open(str(parameter_file), "r") as file_handle: for param in yaml.safe_load(file_handle): parameters[param["ParameterKey"]] = param["ParameterValue"] tests[test_name] = { "template": "templates/" + test_data.template_file, "parameters": parameters, "regions": test_data.regions, } if not tests[test_name]["regions"]: del tests[test_name]["regions"] new_config_dict = { "project": { "name": legacy_config.global_.qsname, "owner": legacy_config.global_.owner, "s3_bucket": legacy_config.global_.s3bucket, "package_lambda": legacy_config.global_.lambda_build, "regions": legacy_config.global_.regions, }, "tests": tests, } new_config = BaseConfig.from_dict(new_config_dict) LOG.warning( "config is in a legacy format, support for which will be dropped in a " "future version. a new format config (.taskcat.yml) will been placed " "in your project_root" ) new_config_path = project_root / ".taskcat.yml" if new_config_path.exists(): LOG.warning( f"skipping new config file creation, file already exits at " f"{new_config_path}" ) else: with open(str(new_config_path), "w") as file_handle: config_dict = new_config.to_dict() config_dict.pop("general") yaml.dump(config_dict, file_handle, default_flow_style=False) return new_config
def _absolute_path(self, path: Optional[Union[str, Path]]) -> Optional[Path]: if path is None: return path path = Path(path) abs_path = absolute_path(path) if self.project_root and not abs_path: abs_path = absolute_path(Path(self.project_root) / Path(path)) if not abs_path: raise TaskCatException( f"Unable to resolve path {path}, with project_root " f"{self.project_root}") return abs_path
def clean_up(self) -> None: # noqa: C901 """Deletes the Test related resources in AWS. Raises: TaskCatException: If one or more stacks failed to create. """ if not hasattr(self, "test_definition"): LOG.warning("No stacks were created... skipping cleanup.") return status = self.test_definition.status() # Delete Stacks if self.no_delete: LOG.info("Skipping delete due to cli argument") elif self.keep_failed: if len(status["COMPLETE"]) > 0: LOG.info("deleting successful stacks") self.test_definition.delete_stacks( {"status": "CREATE_COMPLETE"}) else: self.test_definition.delete_stacks() if not self.dont_wait_for_delete: self.printer.report_test_progress(stacker=self.test_definition) # TODO: summarise stack statusses (did they complete/delete ok) and print any # error events # Delete Templates and Buckets buckets = self.config.get_buckets() if not self.no_delete or (self.keep_failed is True and len(status["FAILED"]) == 0): deleted: ListType[str] = [] for test in buckets.values(): for bucket in test.values(): if (bucket.name not in deleted) and not bucket.regional_buckets: bucket.delete(delete_objects=True) deleted.append(bucket.name) # 9. raise if something failed # - grabbing the status again to ensure everything deleted OK. status = self.test_definition.status() if len(status["FAILED"]) > 0: raise TaskCatException( f'One or more stacks failed to create: {status["FAILED"]}')
def _bucket_matches_existing(self): try: location = self.s3_client.get_bucket_location( Bucket=self.name)["LocationConstraint"] location = location if location else "us-east-1" except self.s3_client.exceptions.NoSuchBucket: location = None if location != self.region and location is not None: raise TaskCatException( f"bucket {self.name} already exists, but is not in " f"the expected region {self.region}, expected {location}") if location: tags = self.s3_client.get_bucket_tagging( Bucket=self.name)["TagSet"] tags = {t["Key"]: t["Value"] for t in tags} uid = tags.get("taskcat-id") uid = uuid.UUID(uid) if uid else uid if uid != self.taskcat_id: raise TaskCatException( f"bucket {self.name} already exists, but does not have a matching" f" uuid") return True return False
def __init__(self, input_file: str, project_root: str = "./", strict: bool = False): """ :param input_file: path to project config or CloudFormation template :param project_root: base path for project :param strict: fail on lint warnings as well as errors """ config = Config( project_config_path=input_file, project_root=project_root, create_clients=False, ) lint = TaskCatLint(config, strict) errors = lint.lints[1] lint.output_results() if errors or not lint.passed: raise TaskCatException("Lint failed with errors")
def _get_file_list(self, input_path): if not self._file_list: _file_list = [] if os.path.isfile(input_path): _file_list.append(input_path) elif os.path.isdir(input_path): for root, dirs, files in os.walk(input_path): for _current_file in files: if not _current_file.endswith(tuple(self._GIT_EXT)): _file_list.append(os.path.join(root, _current_file)) for directory in self._EXCLUDED_DIRS: if directory in dirs: dirs.remove(directory) else: raise TaskCatException("Directory/File is non-existent. Aborting.") self._file_list = _file_list return self._file_list
def _sync(self, local_list, s3_list, bucket, prefix, acl, threads=16): # determine which files to remove from S3 remove_from_s3 = [] for s3_file in s3_list.keys(): if s3_file not in local_list.keys( ) and not self._exclude_remote(s3_file): print("{}[S3: DELETE ]{} s3://{}/{}".format( PrintMsg.white, PrintMsg.rst_color, bucket, prefix + prefix + s3_file)) remove_from_s3.append({"Key": prefix + s3_file}) # deleting objects, max 1k objects per s3 delete_objects call for d in [ remove_from_s3[i:i + 1000] for i in range(0, len(remove_from_s3), 1000) ]: response = self.s3_client.delete_objects(Bucket=bucket, Delete={'Objects': d}) if "Errors" in response.keys(): for error in response["Errors"]: print(PrintMsg.ERROR + "S3 delete error: %s" % str(error)) raise TaskCatException( "Failed to delete one or more files from S3") # build list of files to upload upload_to_s3 = [] for local_file in local_list: upload = False # If file is not present in S3 if local_file not in s3_list.keys(): upload = True # If checksum is different elif local_list[local_file][1] != s3_list[local_file]: upload = True if upload: absolute_path = local_list[local_file][0] s3_path = local_file upload_to_s3.append([absolute_path, bucket, s3_path]) # multithread the uploading of files pool = ThreadPool(threads) func = partial(self._s3_upload_file, prefix=prefix, s3_client=self.s3_client, acl=acl) pool.map(func, upload_to_s3) pool.close() pool.join()
def validate_all_templates(config: Config) -> None: _validated_templates: List[Path] = [] for test in config.tests.values(): if test.template_file in _validated_templates: continue try: region = test.regions[0] test.template.validate(region.name, region.bucket.name) # type: ignore _validated_templates.append(test.template_file) LOG.info( f"Validated template: {str(test.template_file)}", extra={"nametag": PrintMsg.PASS}, ) except Exception as e: LOG.critical(f"Exception: {str(e)}") raise TaskCatException( f"Unable to validate {test.template_file.name}")
def _recurse_nodes(self, current_node): if type(current_node) in [OrderedDict, dict]: for key in current_node.keys(): self.logger.debug("Key: ") self.logger.debug(key) self.logger.debug("Type: ") self.logger.debug(type(current_node[key])) self.logger.debug("Value: ") self.logger.debug(current_node[key]) current_node[key] = self._recurse_nodes(current_node[key]) elif type(current_node) is list: for _index, item in enumerate(current_node): self.logger.debug("Type: ") self.logger.debug(type(item)) self.logger.debug("Value: ") self.logger.debug(item) current_node[_index] = self._recurse_nodes(item) return current_node elif type(current_node) is str: return self._string_rewriter(current_node, self._target_bucket_name) elif type(current_node) is bool: self.logger.debug("Not much we can do with booleans. Skipping.") elif type(current_node) in [int, float]: self.logger.debug("Not much we can do with numbers. Skipping.") elif type(current_node) in [ datetime.date, datetime.time, datetime.datetime, datetime.timedelta ]: self.logger.debug("Not much we can do with datetime. Skipping.") elif type(current_node) is None: self.logger.debug("Not much we can do with nulls. Skipping.") else: self.logger.error("Unsupported type.") self.logger.error("Failing Type: ") self.logger.error(type(current_node)) self.logger.error("Failing Value: ") self.logger.error(current_node) raise TaskCatException("Unsupported type.") self.logger.debug("PARSED!") return current_node
def _propagate_regions(self, test: Test): # TODO: Better way to handle default_region default_region = test.client_factory.get_default_region( None, None, None, None) if not test.regions and not default_region and not self.regions: raise TaskCatException( f"unable to define region for test {test.name}, you must define " f"regions " f"or set a default region in the aws cli") if not test.regions: if self.regions: test.regions = [ AWSRegionObject(region, self._client_factory_instance) for region in self.regions ] else: test.regions = [ AWSRegionObject(default_region, self._client_factory_instance) ]
def _s3_upload_file(self, paths, prefix, s3_client, acl): local_filename, bucket, s3_path = paths retry = 0 # backoff and retry while retry < 5: print("{}[S3: -> ]{} s3://{}/{}".format(PrintMsg.white, PrintMsg.rst_color, bucket, prefix + s3_path)) try: s3_client.upload_file(local_filename, bucket, prefix + s3_path, ExtraArgs={'ACL': acl}) break except Exception as e: retry += 1 print(PrintMsg.ERROR + "S3 upload error: %s" % e) # give up if we've exhausted retries, or if the error is not-retryable (ie AccessDenied) if retry == 5 or (type(e) == S3UploadFailedError and '(AccessDenied)' in str(e)): raise TaskCatException("Failed to upload to S3") time.sleep(retry * 2)
def _s3_upload_file(paths, prefix, s3_client, acl): local_filename, bucket, s3_path = paths retry = 0 # backoff and retry while retry < 5: LOG.info(f"s3://{bucket}/{prefix + s3_path}", extra={"nametag": PrintMsg.S3}) try: s3_client.upload_file(local_filename, bucket, prefix + s3_path, ExtraArgs={"ACL": acl}) break except Exception as e: # pylint: disable=broad-except retry += 1 LOG.error("S3 upload error: %s" % e) # give up if we've exhausted retries, or if the error is not-retryable # ie. AccessDenied if retry == 5 or (isinstance(e, S3UploadFailedError) and "(AccessDenied)" in str(e)): raise TaskCatException("Failed to upload to S3") time.sleep(retry * 2)
def __init__( self, input_file: str = ".taskcat.yml", project_root: str = "./", strict: bool = False, ): """ :param input_file: path to project config or CloudFormation template :param project_root: base path for project :param strict: fail on lint warnings as well as errors """ project_root_path: Path = Path(project_root).expanduser().resolve() input_file_path: Path = project_root_path / input_file config = Config.create(project_root=project_root_path, project_config_path=input_file_path) templates = config.get_templates() lint = TaskCatLint(config, templates, strict) errors = lint.lints[1] lint.output_results() if errors or not lint.passed: raise TaskCatException("Lint failed with errors")
def _get_presigned_url_wrapper(self, presigned_url_regex): if presigned_url_regex.search(self.param_value): if len(self.param_value) < 2: LOG.error("Syntax: $[taskcat_presignedurl],bucket,key,OPTIONAL_TIMEOUT") raise TaskCatException( "Syntax error when using $[taskcat_getpresignedurl]; Not " "enough parameters." ) paramsplit = self.regxfind(presigned_url_regex, self.param_value).split( "," )[1:] url_bucket, url_key = paramsplit[:2] if len(paramsplit) == 3: url_expire_seconds = paramsplit[2] else: url_expire_seconds = 3600 s3_client = self._boto_client("s3") param_value = s3_client.generate_presigned_url( "get_object", Params={"Bucket": url_bucket, "Key": url_key}, ExpiresIn=int(url_expire_seconds), ) self._regex_replace_param_value(re.compile("^.*$"), param_value) self._regex_replace_param_value(re.compile("^.*$"), param_value)
def _find_children(self) -> None: # noqa: C901 children = set() if "Resources" not in self.template: raise TaskCatException( f"did not receive a valid template: {self.template_path} does not " f"have a Resources section" ) for resource in self.template["Resources"].keys(): resource = self.template["Resources"][resource] if resource["Type"] == "AWS::CloudFormation::Stack": child_name = self._template_url_to_path( template_url=resource["Properties"]["TemplateURL"], ) # print(child_name) if child_name: # for child_url in child_name: children.add(child_name) for child in children: child_template_instance = None for descendent in self.descendents: if str(descendent.template_path) == str(child): child_template_instance = descendent if not child_template_instance: try: child_template_instance = Template( child, self.project_root, self._get_relative_url(child), self._s3_key_prefix, tcat_template_cache, ) except Exception: # pylint: disable=broad-except LOG.debug("Traceback:", exc_info=True) LOG.error(f"Failed to add child template {child}") if isinstance(child_template_instance, Template): self.children.append(child_template_instance)
def run( input_file: str = "./.taskcat.yml", project_root: str = "./", no_delete: bool = False, lint_disable: bool = False, enable_sig_v2: bool = False, keep_failed: bool = False, ): """tests whether CloudFormation templates are able to successfully launch :param input_file: path to either a taskat project config file or a CloudFormation template :param project_root_path: root path of the project relative to input_file :param no_delete: don't delete stacks after test is complete :param lint_disable: disable cfn-lint checks :param enable_sig_v2: enable legacy sigv2 requests for auto-created buckets :param keep_failed: do not delete failed stacks """ project_root_path: Path = Path(project_root).expanduser().resolve() input_file_path: Path = project_root_path / input_file config = Config.create( project_root=project_root_path, project_config_path=input_file_path # TODO: detect if input file is taskcat config or CloudFormation template ) if enable_sig_v2: config = Config.create( project_root=project_root_path, project_config_path=input_file_path, args={"project": { "s3_enable_sig_v2": enable_sig_v2 }}, ) boto3_cache = Boto3Cache() templates = config.get_templates(project_root_path) # 1. lint if not lint_disable: lint = TaskCatLint(config, templates) errors = lint.lints[1] lint.output_results() if errors or not lint.passed: raise TaskCatException("Lint failed with errors") # 2. build lambdas LambdaBuild(config, project_root_path) # 3. s3 sync buckets = config.get_buckets(boto3_cache) stage_in_s3(buckets, config.config.project.name, project_root_path) # 4. launch stacks regions = config.get_regions(boto3_cache) parameters = config.get_rendered_parameters(buckets, regions, templates) tests = config.get_tests(project_root_path, templates, regions, buckets, parameters) test_definition = Stacker( config.config.project.name, tests, shorten_stack_name=config.config.project.shorten_stack_name, ) test_definition.create_stacks() terminal_printer = TerminalPrinter() # 5. wait for completion terminal_printer.report_test_progress(stacker=test_definition) status = test_definition.status() # 6. create report report_path = Path("./taskcat_outputs/").resolve() report_path.mkdir(exist_ok=True) cfn_logs = _CfnLogTools() cfn_logs.createcfnlogs(test_definition, report_path) ReportBuilder(test_definition, report_path / "index.html").generate_report() # 7. delete stacks if no_delete: LOG.info("Skipping delete due to cli argument") elif keep_failed: if len(status["COMPLETE"]) > 0: LOG.info("deleting successful stacks") test_definition.delete_stacks({"status": "CREATE_COMPLETE"}) terminal_printer.report_test_progress(stacker=test_definition) else: test_definition.delete_stacks() terminal_printer.report_test_progress(stacker=test_definition) # TODO: summarise stack statusses (did they complete/delete ok) and print any # error events # 8. delete buckets if not no_delete or (keep_failed is True and len(status["FAILED"]) == 0): deleted: ListType[str] = [] for test in buckets.values(): for bucket in test.values(): if bucket.name not in deleted: bucket.delete(delete_objects=True) deleted.append(bucket.name) # 9. raise if something failed if len(status["FAILED"]) > 0: raise TaskCatException( f'One or more stacks failed tests: {status["FAILED"]}')
def __init__( # noqa: C901 self, package: str, aws_profile: str = "default", region="default", parameters="", name="", wait=False, ): """ :param package: name of package to install can be a path to a local package, a github org/repo, or an AWS Quick Start name :param aws_profile: aws profile to use for installation :param region: regions to install into, default will use aws cli configured default :param parameters: parameters to pass to the stack, in the format Key=Value,AnotherKey=AnotherValue or providing a path to a json or yaml file containing the parameters :param name: stack name to use, if not specified one will be automatically generated :param wait: if enabled, taskcat will wait for stack to complete before exiting """ LOG.warning("deploy is in alpha feature, use with caution") boto3_cache = Boto3Cache() if not name: name = generate_name() if region == "default": region = boto3_cache.get_default_region(profile_name=aws_profile) path = Path(package).resolve() if Path(package).resolve().is_dir(): package_type = "local" elif "/" in package: package_type = "github" else: # assuming it's an AWS Quick Start package_type = "github" package = f"aws-quickstart/quickstart-{package}" if package_type == "github": if package.startswith("https://") or package.startswith("git@"): url = package org, repo = (package.replace(".git", "").replace(":", "/").split("/")[-2:]) else: org, repo = package.split("/") url = f"https://github.com/{org}/{repo}.git" path = Deploy.PKG_CACHE_PATH / org / repo LOG.info(f"fetching git repo {url}") self._git_clone(url, path) self._recurse_submodules(path, url) config = Config.create( args={"project": { "regions": [region] }}, project_config_path=(path / ".taskcat.yml"), project_root=path, ) # only use one region for test_name in config.config.tests: config.config.tests[ test_name].regions = config.config.project.regions # if there's no test called default, take the 1st in the list if "default" not in config.config.tests: config.config.tests["default"] = config.config.tests[list( config.config.tests.keys())[0]] # until install offers a way to run different "plans" we only need one test for test_name in list(config.config.tests.keys()): if test_name != "default": del config.config.tests[test_name] buckets = config.get_buckets(boto3_cache) stage_in_s3(buckets, config.config.project.name, path) regions = config.get_regions(boto3_cache) templates = config.get_templates(project_root=path) parameters = config.get_rendered_parameters(buckets, regions, templates) tests = config.get_tests(path, templates, regions, buckets, parameters) tags = [Tag({"Key": "taskcat-installer", "Value": name})] stacks = Stacker(config.config.project.name, tests, tags=tags) stacks.create_stacks() LOG.error( f" {stacks.uid.hex}", extra={"nametag": "\x1b[0;30;47m[INSTALL_ID ]\x1b[0m"}, ) LOG.error(f" {name}", extra={"nametag": "\x1b[0;30;47m[INSTALL_NAME]\x1b[0m"}) if wait: LOG.info( f"waiting for stack {stacks.stacks[0].name} to complete in " f"{stacks.stacks[0].region_name}") while stacks.status()["IN_PROGRESS"]: sleep(5) if stacks.status()["FAILED"]: LOG.error("Install failed:") for error in stacks.stacks[0].error_events(): LOG.error(f"{error.logical_id}: {error.status_reason}") raise TaskCatException("Stack creation failed")
def run( # noqa: C901 test_names: str = "ALL", regions: str = "ALL", input_file: str = "./.taskcat.yml", project_root: str = "./", no_delete: bool = False, lint_disable: bool = False, enable_sig_v2: bool = False, keep_failed: bool = False, output_directory: str = "./taskcat_outputs", minimal_output: bool = False, dont_wait_for_delete: bool = False, ): """tests whether CloudFormation templates are able to successfully launch :param test_names: comma separated list of tests to run :param regions: comma separated list of regions to test in :param input_file: path to either a taskat project config file or a CloudFormation template :param project_root_path: root path of the project relative to input_file :param no_delete: don't delete stacks after test is complete :param lint_disable: disable cfn-lint checks :param enable_sig_v2: enable legacy sigv2 requests for auto-created buckets :param keep_failed: do not delete failed stacks :param output_directory: Where to store generated logfiles :param minimal_output: Reduces output during test runs :param dont_wait_for_delete: Exits immediately after calling stack_delete """ project_root_path: Path = Path(project_root).expanduser().resolve() input_file_path: Path = project_root_path / input_file # pylint: disable=too-many-arguments args = _build_args(enable_sig_v2, regions, GLOBAL_ARGS.profile) config = Config.create( project_root=project_root_path, project_config_path=input_file_path, args=args # TODO: detect if input file is taskcat config or CloudFormation template ) _trim_regions(regions, config) _trim_tests(test_names, config) boto3_cache = Boto3Cache() templates = config.get_templates() # 1. lint if not lint_disable: lint = TaskCatLint(config, templates) errors = lint.lints[1] lint.output_results() if errors or not lint.passed: raise TaskCatException("Lint failed with errors") # 2. build lambdas if config.config.project.package_lambda: LambdaBuild(config, project_root_path) # 3. s3 sync buckets = config.get_buckets(boto3_cache) stage_in_s3(buckets, config.config.project.name, config.project_root) # 4. launch stacks regions = config.get_regions(boto3_cache) parameters = config.get_rendered_parameters(buckets, regions, templates) tests = config.get_tests(templates, regions, buckets, parameters) test_definition = Stacker( config.config.project.name, tests, shorten_stack_name=config.config.project.shorten_stack_name, ) test_definition.create_stacks() terminal_printer = TerminalPrinter(minimalist=minimal_output) # 5. wait for completion terminal_printer.report_test_progress(stacker=test_definition) status = test_definition.status() # 6. create report report_path = Path(output_directory).resolve() report_path.mkdir(exist_ok=True) cfn_logs = _CfnLogTools() cfn_logs.createcfnlogs(test_definition, report_path) ReportBuilder(test_definition, report_path / "index.html").generate_report() # 7. delete stacks if no_delete: LOG.info("Skipping delete due to cli argument") elif keep_failed: if len(status["COMPLETE"]) > 0: LOG.info("deleting successful stacks") test_definition.delete_stacks({"status": "CREATE_COMPLETE"}) if not dont_wait_for_delete: terminal_printer.report_test_progress( stacker=test_definition) else: test_definition.delete_stacks() if not dont_wait_for_delete: terminal_printer.report_test_progress(stacker=test_definition) # TODO: summarise stack statusses (did they complete/delete ok) and print any # error events # 8. delete buckets if not no_delete or (keep_failed is True and len(status["FAILED"]) == 0): deleted: ListType[str] = [] for test in buckets.values(): for bucket in test.values(): if (bucket.name not in deleted) and not bucket.regional_buckets: bucket.delete(delete_objects=True) deleted.append(bucket.name) # 9. raise if something failed if len(status["FAILED"]) > 0: raise TaskCatException( f'One or more stacks failed tests: {status["FAILED"]}')
def __init__( # noqa: C901 self, args: Optional[dict] = None, global_config_path: str = "~/.taskcat.yml", project_config_path: Optional[Union[Path, str]] = None, project_root: str = "./", override_file: str = None, # pylint: disable=unused-argument all_env_vars: Optional[List[dict]] = None, create_clients: bool = True, ): # #pylint: disable=too-many-arguments # #pylint: disable=too-many-statements # inputs if absolute_path( project_config_path) and not Path(project_root).is_absolute(): project_root = absolute_path( project_config_path).parent / project_root self.project_root: Union[Path, str] = absolute_path(project_root) if not self.project_root: raise TaskCatException( f"project_root {project_root} is not a valid path") self.args: dict = args if args else {} self.global_config_path: Optional[Path] = absolute_path( global_config_path) self._client_factory_instance = ClientFactory() # Used only in initial client configuration, then set to None # general config self.profile_name: str = "" self.aws_access_key: str = "" self.aws_secret_key: str = "" self.no_cleanup: bool = False self.no_cleanup_failed: bool = False self.verbosity: str = "DEBUG" self.tags: dict = {} self.stack_prefix: str = "" self.lint: bool = False self.upload_only: bool = False self.lambda_build_only: bool = False self.exclude: str = "" self.enable_sig_v2: bool = False self.auth: Dict[str, dict] = {} # project config self.name: str = "" self.owner: str = "" self.package_lambda: bool = True self.s3bucket: S3BucketConfig = S3BucketConfig() self.tests: Dict[str, Test] = {} self.regions: Set[str] = set() self.env_vars: Dict[str, str] = {} self.project_config_path: Optional[Path] = None self.template_path: Optional[Path] = None self.lambda_source_path: Path = (Path(self.project_root) / "functions/source/").resolve() self.lambda_zip_path: Path = (Path(self.project_root) / "functions/packages/").resolve() self.build_submodules = True self._harvest_env_vars( all_env_vars if all_env_vars else os.environ.items()) self._process_global_config() if not self._absolute_path(project_config_path): for path in Config.DEFAULT_PROJECT_PATHS: try: project_config_path = self._absolute_path(path) LOG.debug("found project config in default location %s", path) break except TaskCatException: LOG.debug("didn't find project config in %s", path) if not self._absolute_path(project_config_path): raise TaskCatException( f"failed to load project config file {project_config_path}. file " f"does not exist") if self._is_template(self._absolute_path(project_config_path)): self.template_path = self._absolute_path(project_config_path) self._process_template_config() else: self.project_config_path = self._absolute_path(project_config_path) self._process_project_config() self._process_env_vars() self._process_args() if not self.template_path and not self.tests: raise TaskCatException( "minimal config requires at least one test or a " "template_path to be defined") # Add test/region specific credential sets to ClientFactory instance. self._add_granular_credsets_to_cf() # Used where full Regional/S3Bucket properties are needd. # - ie: 'test' subcommand, etc. if create_clients: # Assign regional and test-specific client-factories self._enable_regional_creds() # Assign account-based buckets. self._assign_account_buckets() # generate regional-based parameters self._generate_regional_parameters() # add `template` attribute to Test objects for _, test_obj in self.tests.items(): test_obj.template = None # type: ignore # build and attach template objects self._get_templates()
def _status(stack: Stack): for status_group in ["COMPLETE", "IN_PROGRESS", "FAILED"]: if stack.status in getattr(StackStatus, status_group): return stack.id, status_group, stack.status_reason raise TaskCatException(f"Invalid stack {stack}")
def upload_only(self): """ This function uploads all assets to the target S3 bucket name using the target S3 key prefix for each object. A comparison of checksums is done for all object as well to avoid reuploading files that have not changed (this checksum comparison is only effective on non-multi part uploaded files). """ if self._target_key_prefix is None: raise TaskCatException('target_key_prefix cannot be None') # TODO: FIGURE OUT BOTO SESSION HANDLING DETAILS CURRENTLY USING ClientFactory's get_session from utils.py ''' # Use a profile if args.profile: boto_session = boto3.Session(profile_name=args.profile) s3_resource = boto_session.resource('s3') # Use explicit credentials elif args.access_key_id and args.secret_access_key: boto_session = boto3.Session(aws_access_key_id=args.access_key_id, aws_secret_access_key=args.secret_access_key) s3_resource = boto_session.resource('s3') # Attempt to use IAM role from instance profile else: boto_session = boto3.Session() s3_resource = boto_session.resource('s3') ''' boto_session = self._boto_clients.get_session( credential_set='alchemist', region=self.get_default_region() ) s3_resource = boto_session.resource('s3') upload_bucket = s3_resource.Bucket(self._target_bucket_name) self.logger.info("Gathering remote S3 bucket keys {}*".format(self._target_key_prefix)) remote_key_dict = {} for obj in upload_bucket.objects.filter(Prefix='{}'.format(self._target_key_prefix)): if any(x not in obj.key for x in self._get_excluded_key_prefixes()): remote_key_dict[obj.key] = obj self.logger.debug(remote_key_dict.keys()) # Gather file list # NOTE: We only use the output directory if it's been set (that is, a rewrite was expected to have happened to # an output directory. We ensure that is not the case when parsing the args, but care must be taken # when initializing all the properties of this class. If it's only an upload that's meant to happen # without a previous rewrite, then output directory should never be set. self.logger.info("Gathering local keys {}*".format(self._target_key_prefix)) if self._file_list: file_list = self._file_list else: file_list = self._get_file_list(self._input_path) local_key_dict = {} for current_file in file_list: local_key_dict[os.path.join(self._target_key_prefix, current_file.replace(self._input_path, '', 1).lstrip('\/')).replace('\\', '/')] = \ os.path.join(self._output_directory if self._output_directory and not self._dry_run else self._input_path, current_file.replace(self._input_path, '', 1).lstrip('\/')) self.logger.debug(local_key_dict.keys()) remote_to_local_diff = list(set(remote_key_dict.keys()) - set(local_key_dict.keys())) self.logger.info("Keys in remote S3 bucket but not in local:") self.logger.info(remote_to_local_diff) local_to_remote_diff = list(set(local_key_dict.keys()) - set(remote_key_dict.keys())) self.logger.info("Keys in local but not in remote S3 bucket:") self.logger.info(local_to_remote_diff) self.logger.info("Syncing objects to S3 bucket [{}]".format(self._target_bucket_name)) for _key in local_key_dict.keys(): if _key in remote_key_dict: self.logger.debug("File [{0}] exists in S3 bucket [{1}]. Verifying MD5 checksum for difference.".format(_key, self._target_bucket_name)) s3_hash = remote_key_dict[_key].e_tag.strip('"') local_hash = hashlib.md5(open(local_key_dict[_key], 'rb').read()).hexdigest() self.logger.debug("S3 MD5 checksum (etag) [{0}]=>[{1}]".format(s3_hash, remote_key_dict[_key])) self.logger.debug("Local MD5 checksum [{0}]=>[{1}]".format(local_hash, local_key_dict[_key])) if s3_hash != local_hash: if self._dry_run: self.logger.info("[WHAT IF DRY RUN]: UPDATE [{0}]".format(_key)) else: self.logger.info("UPDATE [{0}]".format(_key)) s3_resource.Object(self._target_bucket_name, _key).upload_file(local_key_dict[_key]) else: self.logger.debug("MD5 checksums are the same. Skipping [{}]".format(_key)) else: if self._dry_run: self.logger.info("[WHAT IF DRY RUN]: CREATE [{0}]".format(_key)) else: self.logger.info("CREATE [{0}]".format(_key)) # Upload local file not present in S3 bucket s3_resource.Object(self._target_bucket_name, _key).upload_file(local_key_dict[_key]) # clean up/remove remote keys that are not in local keys for _key in remote_to_local_diff: if not any(x in _key for x in self._get_excluded_key_prefixes()): if self._dry_run: self.logger.info("[WHAT IF DRY RUN]: DELETE [{0}]".format(_key)) else: self.logger.info("DELETE [{0}]".format(_key)) remote_key_dict[_key].delete()
def get_s3_domain(region): try: return S3_PARTITION_MAP[REGIONS[region]] except KeyError: raise TaskCatException( f"cannot find the S3 hostname for region {region}")