def handle_action_execution_detail(puppet_account_id, action_execution_detail): action_type_id = action_execution_detail.get("input").get("actionTypeId") if ( action_type_id.get("category") == "Build" and action_type_id.get("owner") == "AWS" and action_type_id.get("provider") == "CodeBuild" ): external_execution_id = ( action_execution_detail.get("output") .get("executionResult") .get("externalExecutionId") ) with betterboto_client.ClientContextManager( "codebuild", region_name=config.get_home_region(puppet_account_id) ) as codebuild: builds = codebuild.batch_get_builds(ids=[external_execution_id]).get( "builds" ) build = builds[0] log_details = build.get("logs") with betterboto_client.ClientContextManager( "logs", region_name=config.get_home_region(puppet_account_id) ) as logs: with open( f"log-{action_execution_detail.get('input').get('configuration').get('ProjectName')}.log", "w", ) as f: params = { "logGroupName": log_details.get("groupName"), "logStreamName": log_details.get("streamName"), "startFromHead": True, } has_more_logs = True while has_more_logs: get_log_events_response = logs.get_log_events(**params) if (len(get_log_events_response.get("events"))) > 0: params["nextToken"] = get_log_events_response.get( "nextForwardToken" ) else: has_more_logs = False if params.get("nextToken"): del params["nextToken"] for e in get_log_events_response.get("events"): d = datetime.utcfromtimestamp( e.get("timestamp") / 1000 ).strftime("%Y-%m-%d %H:%M:%S") f.write(f"{d} : {e.get('message')}")
def run(self): home_region = config.get_home_region(self.puppet_account_id) with betterboto_client.CrossAccountClientContextManager( "lambda", f"arn:aws:iam::{self.puppet_account_id}:role/servicecatalog-puppet/PuppetRole", f"sc-{home_region}-{self.puppet_account_id}", region_name=home_region, ) as lambda_client: payload = dict( account_id=self.account_id, region=self.region, parameters=self.get_all_params(), ) response = lambda_client.invoke( FunctionName=self.function_name, InvocationType=self.invocation_type, Payload=json.dumps(payload), Qualifier=self.qualifier, ) success_results = dict(RequestResponse=200, Event=202, DryRun=204) if success_results.get(self.invocation_type) != response.get('StatusCode'): raise Exception(f"{self.lambda_invocation_name} failed for {self.account_id}, {self.region}") else: if response.get('FunctionError'): error_payload = response.get('Payload').read() raise Exception(error_payload) else: output = dict(**self.params_for_results_display(), payload=payload, response=response) self.write_output(output)
def requires(self): all_params = dict() all_params.update(self.manifest_parameters) all_params.update(self.launch_parameters) all_params.update(self.account_parameters) ssm_params = dict() for param_name, param_details in all_params.items(): if param_details.get("ssm"): if param_details.get("default"): del param_details["default"] ssm_params[param_name] = tasks.GetSSMParamTask( parameter_name=param_name, name=param_details.get("ssm").get("name"), region=param_details.get("ssm").get( "region", config.get_home_region(self.puppet_account_id)), cache_invalidator=self.cache_invalidator, ) self.all_params = all_params return dict( ssm_params=ssm_params, dependencies=LambdaInvocationDependenciesWrapperTask( lambda_invocation_name=self.lambda_invocation_name, manifest_file_path=self.manifest_file_path, puppet_account_id=self.puppet_account_id, should_use_sns=self.should_use_sns, should_use_product_plans=self.should_use_product_plans, include_expanded_from=self.include_expanded_from, single_account=self.single_account, is_dry_run=self.is_dry_run, cache_invalidator=self.cache_invalidator, ), )
def run(self): home_region = config.get_home_region(self.puppet_account_id) with self.hub_regional_client( "lambda", region_name=home_region) as lambda_client: payload = dict( account_id=self.account_id, region=self.region, parameters=self.get_parameter_values(), ) response = lambda_client.invoke( FunctionName=self.function_name, InvocationType=self.invocation_type, Payload=json.dumps(payload), Qualifier=self.qualifier, ) success_results = dict(RequestResponse=200, Event=202, DryRun=204) if success_results.get( self.invocation_type) != response.get("StatusCode"): raise Exception( f"{self.lambda_invocation_name} failed for {self.account_id}, {self.region}" ) else: if response.get("FunctionError"): error_payload = response.get("Payload").read() raise Exception(error_payload) else: output = dict( **self.params_for_results_display(), payload=payload, response=response, ) self.write_output(output)
def release_spoke(puppet_account_id): with betterboto_client.ClientContextManager( "cloudformation", region_name=config.get_home_region( puppet_account_id)) as cloudformation: cloudformation.ensure_deleted( StackName=f"{constants.BOOTSTRAP_STACK_NAME}-spoke")
def get_ssm_parameters(self): ssm_params = dict() all_params = self.get_all_of_the_params() for param_name, param_details in all_params.items(): if param_details.get("ssm"): if param_details.get("default"): del param_details["default"] ssm_parameter_name = param_details.get("ssm").get("name") ssm_parameter_name = ssm_parameter_name.replace( "${AWS::Region}", self.region ) ssm_parameter_name = ssm_parameter_name.replace( "${AWS::AccountId}", self.account_id ) ssm_params[param_name] = GetSSMParamTask( parameter_name=param_name, name=ssm_parameter_name, region=param_details.get("ssm").get( "region", config.get_home_region(self.puppet_account_id) ), ) return ssm_params
def export_puppet_pipeline_logs(execution_id, puppet_account_id): with betterboto_client.ClientContextManager( "codepipeline", region_name=config.get_home_region(puppet_account_id) ) as codepipeline: action_execution_details = codepipeline.list_action_executions( pipelineName=constants.PIPELINE_NAME, filter={"pipelineExecutionId": execution_id}, ).get("actionExecutionDetails") for action_execution_detail in action_execution_details: handle_action_execution_detail(puppet_account_id, action_execution_detail)
def handle_action_execution_detail(action_execution_detail): action_type_id = action_execution_detail.get('input').get('actionTypeId') if action_type_id.get('category') == "Build" and action_type_id.get( 'owner') == "AWS" and action_type_id.get( 'provider') == "CodeBuild": external_execution_id = action_execution_detail.get('output').get( 'executionResult').get('externalExecutionId') with betterboto_client.ClientContextManager( "codebuild", region_name=config.get_home_region()) as codebuild: builds = codebuild.batch_get_builds( ids=[external_execution_id]).get('builds') build = builds[0] log_details = build.get('logs') with betterboto_client.ClientContextManager( "logs", region_name=config.get_home_region()) as logs: with open( f"log-{action_execution_detail.get('input').get('configuration').get('ProjectName')}.log", 'w') as f: params = { 'logGroupName': log_details.get('groupName'), 'logStreamName': log_details.get('streamName'), 'startFromHead': True, } has_more_logs = True while has_more_logs: get_log_events_response = logs.get_log_events(**params) if (len(get_log_events_response.get('events'))) > 0: params['nextToken'] = get_log_events_response.get( 'nextForwardToken') else: has_more_logs = False if params.get('nextToken'): del params['nextToken'] for e in get_log_events_response.get('events'): d = datetime.utcfromtimestamp( e.get('timestamp') / 1000).strftime('%Y-%m-%d %H:%M:%S') f.write(f"{d} : {e.get('message')}")
def requires(self): ssm_params = {} for param_name, param_details in self.parameters.items(): if param_details.get('ssm'): if param_details.get('default'): del param_details['default'] ssm_params[param_name] = tasks.GetSSMParamTask( parameter_name=param_name, name=param_details.get('ssm').get('name'), region=param_details.get('ssm').get( 'region', config.get_home_region())) return { 'ssm_params': ssm_params, }
def requires(self): all_params = {} all_params.update(self.manifest_parameters) all_params.update(self.launch_parameters) all_params.update(self.account_parameters) ssm_params = {} for param_name, param_details in all_params.items(): if param_details.get('ssm'): if param_details.get('default'): del param_details['default'] ssm_params[param_name] = tasks.GetSSMParamTask( parameter_name=param_name, name=param_details.get('ssm').get('name'), region=param_details.get('ssm').get( 'region', config.get_home_region())) self.all_params = all_params version_id = portfoliomanagement.GetVersionIdByVersionName( self.portfolio, self.product, self.version, self.account_id, self.region, ) product_id = portfoliomanagement.GetProductIdByProductName( self.portfolio, self.product, self.account_id, self.region, ) return { 'ssm_params': ssm_params, 'version': version_id, 'product': product_id, 'provisioning_artifact_parameters': ProvisioningArtifactParametersTask( self.portfolio, self.product, self.version, self.account_id, self.region, ), }
def requires(self): all_params = dict() all_params.update(self.manifest_parameters) all_params.update(self.launch_parameters) all_params.update(self.account_parameters) ssm_params = dict() for param_name, param_details in all_params.items(): if param_details.get("ssm"): if param_details.get("default"): del param_details["default"] ssm_params[param_name] = tasks.GetSSMParamTask( parameter_name=param_name, name=param_details.get("ssm").get("name"), region=param_details.get("ssm").get( "region", config.get_home_region(self.puppet_account_id) ), ) self.all_params = all_params return { "ssm_params": ssm_params, }
def test_get_home_region(mocked_betterboto_client): # setup from servicecatalog_puppet import config as sut expected_result = { "us-east-3", } mocked_response = {"Parameter": {"Value": expected_result}} mocked_betterboto_client().__enter__( ).get_parameter.return_value = mocked_response puppet_account_id = "" # exercise actual_result = sut.get_home_region(puppet_account_id) # verify assert actual_result == expected_result args, kwargs = mocked_betterboto_client.call_args assert "ssm" == args[0] assert {} == kwargs args, kwargs = mocked_betterboto_client().__enter__( ).get_parameter.call_args assert args == () assert kwargs == {"Name": constants.HOME_REGION_PARAM_NAME}
def requires(self): all_params = {} all_params.update(self.manifest_parameters) all_params.update(self.launch_parameters) all_params.update(self.account_parameters) ssm_params = {} for param_name, param_details in all_params.items(): if param_details.get('ssm'): if param_details.get('default'): del param_details['default'] ssm_params[param_name] = tasks.GetSSMParamTask( parameter_name=param_name, name=param_details.get('ssm').get('name'), region=param_details.get('ssm').get( 'region', config.get_home_region())) self.all_params = all_params dependencies = [] version_id = portfoliomanagement.GetVersionIdByVersionName( self.portfolio, self.product, self.version, self.account_id, self.region, ) product_id = portfoliomanagement.GetProductIdByProductName( self.portfolio, self.product, self.account_id, self.region, ) for r in self.dependencies: if r.get('status') is not None: if r.get('status') == constants.TERMINATED: raise Exception("Unsupported") new_r = r.get_wrapped() del new_r['status'] dependencies.append(self.__class__(**new_r)) else: dependencies.append(self.__class__(**r)) return { 'dependencies': dependencies, 'ssm_params': ssm_params, 'version': version_id, 'product': product_id, 'provisioning_artifact_parameters': ProvisioningArtifactParametersTask( self.portfolio, self.product, self.version, self.account_id, self.region, ), 'pre_actions': [ portfoliomanagement.ProvisionActionTask(**p) for p in self.pre_actions ], }
def get_parameters_tasks(self): ssm_params = dict() boto3_params = dict() requires = dict( ssm_params=ssm_params, boto3_params=boto3_params, ) all_params = self.get_all_of_the_params() for param_name, param_details in all_params.items(): if param_details.get("ssm"): if param_details.get("default"): del param_details["default"] ssm_parameter_name = param_details.get("ssm").get("name") ssm_parameter_name = ssm_parameter_name.replace( "${AWS::Region}", self.region) ssm_parameter_name = ssm_parameter_name.replace( "${AWS::AccountId}", self.account_id) ssm_params[param_name] = GetSSMParamTask( parameter_name=param_name, name=ssm_parameter_name, region=param_details.get("ssm").get( "region", config.get_home_region(self.puppet_account_id)), path=param_details.get("ssm").get("path", ""), recursive=param_details.get("ssm").get("recursive", True), depends_on=param_details.get("ssm").get("depends_on", []), manifest_file_path=self.manifest_file_path, puppet_account_id=self.puppet_account_id, spoke_account_id=self.account_id, spoke_region=self.region, ) if param_details.get("boto3"): if param_details.get("default"): del param_details["default"] boto3 = param_details.get("boto3") account_id = boto3.get("account_id", self.puppet_account_id).replace( "${AWS::AccountId}", self.account_id) region = boto3.get( "region", config.get_home_region(self.puppet_account_id)).replace( "${AWS::Region}", self.region) boto3_params[param_name] = boto3_task.Boto3Task( account_id=account_id, region=region, client=boto3.get("client"), use_paginator=boto3.get("use_paginator", False), call=boto3.get("call"), arguments=boto3.get("arguments", {}), filter=boto3.get("filter").replace("${AWS::Region}", self.region).replace( "${AWS::AccountId}", self.account_id), requester_task_id=self.task_id, requester_task_family=self.task_family, ) return requires
def requires(self): requirements = dict() regions = config.get_regions(self.puppet_account_id) for launch_name, launch_details in self.manifest.get_launches_items(): portfolio = launch_details.get("portfolio") for region in regions: if requirements.get(region) is None: requirements[region] = dict() regional_details = requirements[region] if regional_details.get(portfolio) is None: regional_details[portfolio] = dict(products=dict()) portfolio_details = regional_details[portfolio] if portfolio_details.get("details") is None: portfolio_details[ "details"] = get_portfolio_by_portfolio_name_task.GetPortfolioByPortfolioName( manifest_file_path=self.manifest_file_path, portfolio=portfolio, puppet_account_id=self.puppet_account_id, account_id=self.puppet_account_id, region=region, ) product = launch_details.get("product") products = portfolio_details.get("products") if products.get(product) is None: products[ product] = get_products_and_provisioning_artifacts_task.GetProductsAndProvisioningArtifactsTask( manifest_file_path=self.manifest_file_path, region=region, portfolio=portfolio, puppet_account_id=self.puppet_account_id, ) params = dict() parameter_by_paths = dict() requirements["parameters"] = params requirements["parameter_by_paths"] = parameter_by_paths home_region = config.get_home_region(self.puppet_account_id) for section in constants.SECTION_NAMES_THAT_SUPPORTS_PARAMETERS: for item_name, item_details in self.manifest.get(section, {}).items(): if item_details.get( "execution") == constants.EXECUTION_MODE_SPOKE: for parameter_name, parameter_details in item_details.get( "parameters", {}).items(): if parameter_details.get("ssm") and str( parameter_details.get("ssm").get( "account_id", "")) == str( self.puppet_account_id): r = parameter_details.get("ssm").get( "region", config.get_home_region(self.puppet_account_id)) name = parameter_details.get("ssm").get("name") path = parameter_details.get("ssm").get("path", "") if path == "": accounts_and_regions = self.manifest.get_account_ids_and_regions_used_for_section_item( self.puppet_account_id, section, item_name) for account_id, regions in accounts_and_regions.items( ): for region in regions: n = name.replace( "${AWS::AccountId}", account_id).replace( "${AWS::Region}", region) params[ f"{parameter_name}||{n}||{r}"] = get_ssm_param_task.GetSSMParamTask( parameter_name=parameter_name, name=n, region=r, default_value=parameter_details .get("ssm").get( "default_value"), path=parameter_details.get( "ssm").get("path", ""), recursive=parameter_details. get("ssm").get( "recursive", True), depends_on=parameter_details. get("ssm").get( "depends_on", []), manifest_file_path=self. manifest_file_path, puppet_account_id=self. puppet_account_id, spoke_account_id=self. puppet_account_id, spoke_region=r, ) else: parameter_by_paths[ path] = get_ssm_param_task.GetSSMParamByPathTask( path=parameter_details.get("ssm").get( "path", ""), recursive=parameter_details.get( "ssm").get("recursive", True), region=parameter_details.get( "ssm").get("recursive", home_region), depends_on=parameter_details.get( "ssm").get("depends_on", []), manifest_file_path=self. manifest_file_path, puppet_account_id=self. puppet_account_id, spoke_account_id=self. puppet_account_id, spoke_region=home_region, ) return requirements
def expand(f, puppet_account_id, single_account, subset=None): click.echo("Expanding") target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"]) assemble_manifest_from_ssm(target_directory) manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios( new_manifest ) if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break items_to_delete = list() for section_name in constants.ALL_SECTION_NAMES: deploy_to_name = constants.DEPLOY_TO_NAMES[section_name] for item_name, item in new_manifest.get(section_name, {}).items(): accounts = list() for deploy_details in item.get(deploy_to_name, {}).get("accounts", []): if str(deploy_details.get("account_id")) == str(single_account): accounts.append(deploy_details) print(f"{item_name}: there are " + str(len(accounts))) if item.get(deploy_to_name).get("accounts"): if len(accounts) > 0: item[deploy_to_name]["accounts"] = accounts else: if item[deploy_to_name].get("tags") or item[deploy_to_name].get( "ous" ): del item[deploy_to_name]["accounts"] else: items_to_delete.append(f"{section_name}:{item_name}") for item_to_delete in items_to_delete: section_name, item_name = item_to_delete.split(":") del new_manifest[section_name][item_name] click.echo("Filtered") new_manifest = manifest_utils.rewrite_cfct(new_manifest) new_manifest = manifest_utils.rewrite_depends_on(new_manifest) new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest) new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id) new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id) new_manifest = manifest_utils.parse_conditions(new_manifest) if subset and subset.get("section"): click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate(new_manifest, subset) manifest_accounts_all = [ {"account_id": a.get("account_id"), "email": a.get("email")} for a in new_manifest.get("accounts", []) ] manifest_accounts_excluding = [ a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id ] # handle all accounts sct_manifest_accounts = json.dumps(manifest_accounts_all) sct_manifest_spokes = json.dumps(manifest_accounts_excluding) regions = config.get_regions(puppet_account_id) sct_config_regions = json.dumps(regions) new_manifest["parameters"]["SCTManifestAccounts"] = dict( default=sct_manifest_accounts ) new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes) new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions) new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() home_region = config.get_home_region(puppet_account_id) with betterboto_client.ClientContextManager("ssm") as ssm: response = ssm.get_parameter(Name="service-catalog-puppet-version") version = response.get("Parameter").get("Value") new_manifest["config_cache"] = dict( home_region=home_region, regions=regions, should_collect_cloudformation_events=config.get_should_use_sns( puppet_account_id, home_region ), should_forward_events_to_eventbridge=config.get_should_use_eventbridge( puppet_account_id, home_region ), should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter( puppet_account_id, home_region ), puppet_version=version, ) new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml_utils.dump(new_manifest))
def expand(f, puppet_account_id, single_account, subset=None): click.echo("Expanding") manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break click.echo("Filtered") new_manifest = manifest_utils.rewrite_depends_on(new_manifest) new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest) new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id) if subset: click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate( manifest_utils.Manifest(new_manifest), subset ) new_manifest = json.loads(json.dumps(new_manifest)) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() home_region = config.get_home_region(puppet_account_id) with betterboto_client.ClientContextManager("ssm") as ssm: response = ssm.get_parameter(Name="service-catalog-puppet-version") version = response.get("Parameter").get("Value") new_manifest["config_cache"] = dict( home_region=home_region, regions=config.get_regions(puppet_account_id, home_region), should_collect_cloudformation_events=config.get_should_use_sns( puppet_account_id, home_region ), should_forward_events_to_eventbridge=config.get_should_use_eventbridge( puppet_account_id, home_region ), should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter( puppet_account_id, home_region ), puppet_version=version, ) new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def uninstall(): with betterboto_client.ClientContextManager( "cloudformation", region_name=config.get_home_region() ) as cloudformation: cloudformation.ensure_deleted(StackName=constants.BOOTSTRAP_STACK_NAME)