def _do_bootstrap(puppet_version, with_manual_approvals): click.echo('Starting bootstrap') should_use_eventbridge = config.get_should_use_eventbridge( os.environ.get("AWS_DEFAULT_REGION")) if should_use_eventbridge: with betterboto_client.ClientContextManager('events') as events: try: events.describe_event_bus(Name=constants.EVENT_BUS_NAME) except events.exceptions.ResourceNotFoundException: events.create_event_bus(Name=constants.EVENT_BUS_NAME, ) all_regions = config.get_regions(os.environ.get("AWS_DEFAULT_REGION")) with betterboto_client.MultiRegionClientContextManager( 'cloudformation', all_regions) as clients: click.echo('Creating {}-regional'.format( constants.BOOTSTRAP_STACK_NAME)) threads = [] template = asset_helpers.read_from_site_packages( '{}.template.yaml'.format('{}-regional'.format( constants.BOOTSTRAP_STACK_NAME))) template = Template(template).render(VERSION=puppet_version) args = { 'StackName': '{}-regional'.format(constants.BOOTSTRAP_STACK_NAME), 'TemplateBody': template, 'Capabilities': ['CAPABILITY_IAM'], 'Parameters': [ { 'ParameterKey': 'Version', 'ParameterValue': puppet_version, 'UsePreviousValue': False, }, { 'ParameterKey': 'DefaultRegionValue', 'ParameterValue': os.environ.get('AWS_DEFAULT_REGION'), 'UsePreviousValue': False, }, ], 'Tags': [{ "Key": "ServiceCatalogPuppet:Actor", "Value": "Framework", }] } for client_region, client in clients.items(): process = Thread(name=client_region, target=client.create_or_update, kwargs=args) process.start() threads.append(process) for process in threads: process.join() click.echo('Finished creating {}-regional'.format( constants.BOOTSTRAP_STACK_NAME)) with betterboto_client.ClientContextManager( 'cloudformation') as cloudformation: click.echo('Creating {}'.format(constants.BOOTSTRAP_STACK_NAME)) template = asset_helpers.read_from_site_packages( '{}.template.yaml'.format(constants.BOOTSTRAP_STACK_NAME)) template = Template(template).render(VERSION=puppet_version, ALL_REGIONS=all_regions) args = { 'StackName': constants.BOOTSTRAP_STACK_NAME, 'TemplateBody': template, 'Capabilities': ['CAPABILITY_NAMED_IAM'], 'Parameters': [ { 'ParameterKey': 'Version', 'ParameterValue': puppet_version, 'UsePreviousValue': False, }, { 'ParameterKey': 'OrgIamRoleArn', 'ParameterValue': str(config.get_org_iam_role_arn()), 'UsePreviousValue': False, }, { 'ParameterKey': 'WithManualApprovals', 'ParameterValue': "Yes" if with_manual_approvals else "No", 'UsePreviousValue': False, }, ], } cloudformation.create_or_update(**args) click.echo('Finished creating {}.'.format(constants.BOOTSTRAP_STACK_NAME)) with betterboto_client.ClientContextManager('codecommit') as codecommit: response = codecommit.get_repository( repositoryName=constants.SERVICE_CATALOG_PUPPET_REPO_NAME) clone_url = response.get('repositoryMetadata').get('cloneUrlHttp') clone_command = "git clone --config 'credential.helper=!aws codecommit credential-helper $@' " \ "--config 'credential.UseHttpPath=true' {}".format(clone_url) click.echo( 'You need to clone your newly created repo now and will then need to seed it: \n{}' .format(clone_command))
def list_launches(f, format): manifest = manifest_utils.load(f) if format == "table": click.echo("Getting details from your account...") all_regions = config.get_regions(os.environ.get("AWS_DEFAULT_REGION")) account_ids = [a.get('account_id') for a in manifest.get('accounts')] deployments = {} for account_id in account_ids: for region_name in all_regions: role = "arn:aws:iam::{}:role/{}".format( account_id, 'servicecatalog-puppet/PuppetRole') logger.info("Looking at region: {} in account: {}".format( region_name, account_id)) with betterboto_client.CrossAccountClientContextManager( 'servicecatalog', role, 'sc-{}-{}'.format(account_id, region_name), region_name=region_name) as spoke_service_catalog: response = spoke_service_catalog.list_accepted_portfolio_shares( ) portfolios = response.get('PortfolioDetails', []) response = spoke_service_catalog.list_portfolios() portfolios += response.get('PortfolioDetails', []) for portfolio in portfolios: portfolio_id = portfolio.get('Id') response = spoke_service_catalog.search_products_as_admin( PortfolioId=portfolio_id) for product_view_detail in response.get( 'ProductViewDetails', []): product_view_summary = product_view_detail.get( 'ProductViewSummary') product_id = product_view_summary.get('ProductId') response = spoke_service_catalog.search_provisioned_products( Filters={ 'SearchQuery': ["productId:{}".format(product_id)] }) for provisioned_product in response.get( 'ProvisionedProducts', []): launch_name = provisioned_product.get('Name') status = provisioned_product.get('Status') provisioning_artifact_response = spoke_service_catalog.describe_provisioning_artifact( ProvisioningArtifactId=provisioned_product.get( 'ProvisioningArtifactId'), ProductId=provisioned_product.get('ProductId'), ).get('ProvisioningArtifactDetail') if deployments.get(account_id) is None: deployments[account_id] = { 'account_id': account_id, constants.LAUNCHES: {} } if deployments[account_id][constants.LAUNCHES].get( launch_name) is None: deployments[account_id][ constants.LAUNCHES][launch_name] = {} deployments[account_id][constants.LAUNCHES][ launch_name][region_name] = { 'launch_name': launch_name, 'portfolio': portfolio.get('DisplayName'), 'product': manifest.get(constants.LAUNCHES, {}).get(launch_name, {}).get('product'), 'version': provisioning_artifact_response.get('Name'), 'active': provisioning_artifact_response.get( 'Active'), 'region': region_name, 'status': status, } output_path = os.path.sep.join([ constants.LAUNCHES_PATH, account_id, region_name, ]) if not os.path.exists(output_path): os.makedirs(output_path) output = os.path.sep.join([ output_path, "{}.json".format(provisioned_product.get('Id')) ]) with open(output, 'w') as f: f.write( json.dumps(provisioned_product, indent=4, default=str)) results = {} tasks = generate_tasks(f) # deployments[account_id][constants.LAUNCHES][launch_name][region_name] for task in tasks: account_id = task.get('account_id') launch_name = task.get('launch_name') if deployments.get(account_id, {}).get(constants.LAUNCHES, {}).get(launch_name) is None: pass else: for region, regional_details in deployments[account_id][ constants.LAUNCHES][launch_name].items(): results[f"{account_id}_{region}_{launch_name}"] = { 'account_id': account_id, 'region': region, 'launch': launch_name, 'portfolio': regional_details.get('portfolio'), 'product': regional_details.get('product'), 'expected_version': task.get('version'), 'actual_version': regional_details.get('version'), 'active': regional_details.get('active'), 'status': regional_details.get('status'), } if format == "table": table = [[ 'account_id', 'region', 'launch', 'portfolio', 'product', 'expected_version', 'actual_version', 'active', 'status', ]] for result in results.values(): table.append([ result.get('account_id'), result.get('region'), result.get('launch'), result.get('portfolio'), result.get('product'), result.get('expected_version'), Color("{green}" + result.get('actual_version') + "{/green}") if result.get('actual_version') == result.get('expected_version') else Color("{red}" + result.get('actual_version') + "{/red}"), Color("{green}" + str(result.get('active')) + "{/green}") if result.get('active') else Color("{red}" + str(result.get('active')) + "{/red}"), Color("{green}" + result.get('status') + "{/green}") if result.get('status') == "AVAILABLE" else Color("{red}" + result.get('status') + "{/red}") ]) click.echo(terminaltables.AsciiTable(table).table) elif format == "json": click.echo(json.dumps( results, indent=4, default=str, )) else: raise Exception(f"Unsupported format: {format}")
def requires(self): requirements = dict() regions = config.get_regions(self.puppet_account_id) for launch_name, launch_details in self.manifest.get_launches_items(): portfolio = launch_details.get("portfolio") for region in regions: if requirements.get(region) is None: requirements[region] = dict() regional_details = requirements[region] if regional_details.get(portfolio) is None: regional_details[portfolio] = dict(products=dict()) portfolio_details = regional_details[portfolio] if portfolio_details.get("details") is None: portfolio_details[ "details"] = get_portfolio_by_portfolio_name_task.GetPortfolioByPortfolioName( manifest_file_path=self.manifest_file_path, portfolio=portfolio, puppet_account_id=self.puppet_account_id, account_id=self.puppet_account_id, region=region, ) product = launch_details.get("product") products = portfolio_details.get("products") if products.get(product) is None: products[ product] = get_products_and_provisioning_artifacts_task.GetProductsAndProvisioningArtifactsTask( manifest_file_path=self.manifest_file_path, region=region, portfolio=portfolio, puppet_account_id=self.puppet_account_id, ) params = dict() parameter_by_paths = dict() requirements["parameters"] = params requirements["parameter_by_paths"] = parameter_by_paths home_region = config.get_home_region(self.puppet_account_id) for section in constants.SECTION_NAMES_THAT_SUPPORTS_PARAMETERS: for item_name, item_details in self.manifest.get(section, {}).items(): if item_details.get( "execution") == constants.EXECUTION_MODE_SPOKE: for parameter_name, parameter_details in item_details.get( "parameters", {}).items(): if parameter_details.get("ssm") and str( parameter_details.get("ssm").get( "account_id", "")) == str( self.puppet_account_id): r = parameter_details.get("ssm").get( "region", config.get_home_region(self.puppet_account_id)) name = parameter_details.get("ssm").get("name") path = parameter_details.get("ssm").get("path", "") if path == "": accounts_and_regions = self.manifest.get_account_ids_and_regions_used_for_section_item( self.puppet_account_id, section, item_name) for account_id, regions in accounts_and_regions.items( ): for region in regions: n = name.replace( "${AWS::AccountId}", account_id).replace( "${AWS::Region}", region) params[ f"{parameter_name}||{n}||{r}"] = get_ssm_param_task.GetSSMParamTask( parameter_name=parameter_name, name=n, region=r, default_value=parameter_details .get("ssm").get( "default_value"), path=parameter_details.get( "ssm").get("path", ""), recursive=parameter_details. get("ssm").get( "recursive", True), depends_on=parameter_details. get("ssm").get( "depends_on", []), manifest_file_path=self. manifest_file_path, puppet_account_id=self. puppet_account_id, spoke_account_id=self. puppet_account_id, spoke_region=r, ) else: parameter_by_paths[ path] = get_ssm_param_task.GetSSMParamByPathTask( path=parameter_details.get("ssm").get( "path", ""), recursive=parameter_details.get( "ssm").get("recursive", True), region=parameter_details.get( "ssm").get("recursive", home_region), depends_on=parameter_details.get( "ssm").get("depends_on", []), manifest_file_path=self. manifest_file_path, puppet_account_id=self. puppet_account_id, spoke_account_id=self. puppet_account_id, spoke_region=home_region, ) return requirements
def get_task_defs_from_details( self, puppet_account_id, include_expanded_from, launch_name, configuration, launch_or_spoke_local_portfolio, ): logger.info( f"get_task_defs_from_details({include_expanded_from}, {launch_name}, {configuration})" ) launch_details = self.get(launch_or_spoke_local_portfolio).get( launch_name) logger.info(launch_details) accounts = self.get("accounts") if launch_details is None: raise Exception(f"launch_details is None for {launch_name}") deploy_to = launch_details.get("deploy_to") task_defs = [] for tag_list_item in deploy_to.get("tags", []): for account in accounts: for tag in account.get("tags", []): if tag == tag_list_item.get("tag"): tag_account_def = deepcopy(configuration) tag_account_def["account_id"] = account.get( "account_id") if include_expanded_from: tag_account_def["expanded_from"] = account.get( "expanded_from") tag_account_def["organization"] = account.get( "organization") tag_account_def["account_parameters"] = account.get( "parameters", {}) regions = tag_list_item.get("regions", "default_region") if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions", ]: for region_enabled in account.get( "regions_enabled"): region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def[ "region"] = region_enabled task_defs.append(region_tag_account_def) elif regions == "default_region": region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def["region"] = account.get( "default_region") task_defs.append(region_tag_account_def) elif regions == "all": all_regions = config.get_regions( puppet_account_id) for region_enabled in all_regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def[ "region"] = region_enabled task_defs.append(region_tag_account_def) else: raise Exception( f"Unsupported regions {regions} setting for launch: {launch_name}" ) elif isinstance(regions, list): for region in regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def["region"] = region task_defs.append(region_tag_account_def) elif isinstance(regions, tuple): for region in regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def["region"] = region task_defs.append(region_tag_account_def) else: raise Exception( f"Unexpected regions of {regions} set for launch {launch_name}" ) for account_list_item in deploy_to.get("accounts", []): for account in accounts: if account.get("account_id") == account_list_item.get( "account_id"): account_account_def = deepcopy(configuration) account_account_def["account_id"] = account.get( "account_id") if include_expanded_from: account_account_def["expanded_from"] = account.get( "expanded_from") account_account_def["organization"] = account.get( "organization") account_account_def["account_parameters"] = account.get( "parameters", {}) regions = account_list_item.get("regions", "default_region") if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions" ]: for region_enabled in account.get( "regions_enabled"): region_account_account_def = deepcopy( account_account_def) region_account_account_def[ "region"] = region_enabled task_defs.append(region_account_account_def) elif regions in ["default_region"]: region_account_account_def = deepcopy( account_account_def) region_account_account_def["region"] = account.get( "default_region") task_defs.append(region_account_account_def) elif regions in ["all"]: all_regions = config.get_regions(puppet_account_id) for region_enabled in all_regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def[ "region"] = region_enabled task_defs.append(region_account_account_def) else: raise Exception( f"Unsupported regions {regions} setting for launch: {launch_name}" ) elif isinstance(regions, list): for region in regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def["region"] = region task_defs.append(region_account_account_def) elif isinstance(regions, tuple): for region in regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def["region"] = region task_defs.append(region_account_account_def) else: raise Exception( f"Unexpected regions of {regions} set for launch {launch_name}" ) return task_defs
def bootstrap( puppet_account_id, with_manual_approvals, puppet_code_pipeline_role_permission_boundary, source_role_permissions_boundary, puppet_generate_role_permission_boundary, puppet_deploy_role_permission_boundary, puppet_provisioning_role_permissions_boundary, cloud_formation_deploy_role_permissions_boundary, deploy_environment_compute_type, spoke_deploy_environment_compute_type, deploy_num_workers, source_provider, owner, repo, branch, poll_for_source_changes, webhook_secret, puppet_role_name, puppet_role_path, scm_connection_arn, scm_full_repository_id, scm_branch_name, scm_bucket_name, scm_object_key, scm_skip_creation_of_repo, should_validate, custom_source_action_git_url, custom_source_action_git_web_hook_ip_address, custom_source_action_custom_action_type_version, custom_source_action_custom_action_type_provider, ): click.echo("Starting bootstrap") should_use_eventbridge = config.get_should_use_eventbridge( puppet_account_id, os.environ.get("AWS_DEFAULT_REGION") ) initialiser_stack_tags = config.get_initialiser_stack_tags() if should_use_eventbridge: with betterboto_client.ClientContextManager("events") as events: try: events.describe_event_bus(Name=constants.EVENT_BUS_NAME) except events.exceptions.ResourceNotFoundException: events.create_event_bus(Name=constants.EVENT_BUS_NAME,) all_regions = config.get_regions( puppet_account_id, os.environ.get("AWS_DEFAULT_REGION") ) with betterboto_client.MultiRegionClientContextManager( "cloudformation", all_regions ) as clients: click.echo("Creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME)) threads = [] template = hub_bootstrap_region.get_template( constants.VERSION, os.environ.get("AWS_DEFAULT_REGION") ).to_yaml(clean_up=True) args = { "StackName": "{}-regional".format(constants.BOOTSTRAP_STACK_NAME), "TemplateBody": template, "Capabilities": ["CAPABILITY_IAM"], "Parameters": [ { "ParameterKey": "Version", "ParameterValue": constants.VERSION, "UsePreviousValue": False, }, { "ParameterKey": "DefaultRegionValue", "ParameterValue": os.environ.get("AWS_DEFAULT_REGION"), "UsePreviousValue": False, }, ], "Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}] + initialiser_stack_tags, } for client_region, client in clients.items(): process = Thread( name=client_region, target=client.create_or_update, kwargs=args ) process.start() threads.append(process) for process in threads: process.join() click.echo( "Finished creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME) ) source_args = {"Provider": source_provider} if source_provider == "CodeCommit": source_args.update( { "Configuration": { "RepositoryName": repo, "BranchName": branch, "PollForSourceChanges": poll_for_source_changes, }, } ) elif source_provider == "GitHub": source_args.update( { "Configuration": { "Owner": owner, "Repo": repo, "Branch": branch, "PollForSourceChanges": poll_for_source_changes, "SecretsManagerSecret": webhook_secret, }, } ) elif source_provider.lower() == "codestarsourceconnection": source_args.update( { "Configuration": { "ConnectionArn": scm_connection_arn, "FullRepositoryId": scm_full_repository_id, "BranchName": scm_branch_name, "OutputArtifactFormat": "CODE_ZIP", "DetectChanges": poll_for_source_changes, }, } ) elif source_provider.lower() == "s3": source_args.update( { "Configuration": { "S3Bucket": scm_bucket_name, "S3ObjectKey": scm_object_key, "PollForSourceChanges": poll_for_source_changes, }, } ) elif source_provider.lower() == "custom": source_args.update( { "Configuration": { "Owner": "Custom", "GitUrl": custom_source_action_git_url, "Branch": branch, "GitWebHookIpAddress": custom_source_action_git_web_hook_ip_address, "CustomActionTypeVersion": custom_source_action_custom_action_type_version, "CustomActionTypeProvider": custom_source_action_custom_action_type_provider, }, } ) template = hub_bootstrap.get_template( constants.VERSION, all_regions, source_args, config.is_caching_enabled( puppet_account_id, os.environ.get("AWS_DEFAULT_REGION") ), with_manual_approvals, scm_skip_creation_of_repo, should_validate, ).to_yaml(clean_up=True) args = { "StackName": constants.BOOTSTRAP_STACK_NAME, "TemplateBody": template, "Capabilities": ["CAPABILITY_NAMED_IAM"], "Parameters": [ { "ParameterKey": "Version", "ParameterValue": constants.VERSION, "UsePreviousValue": False, }, { "ParameterKey": "OrgIamRoleArn", "ParameterValue": str(config.get_org_iam_role_arn(puppet_account_id)), "UsePreviousValue": False, }, { "ParameterKey": "WithManualApprovals", "ParameterValue": "Yes" if with_manual_approvals else "No", "UsePreviousValue": False, }, { "ParameterKey": "PuppetCodePipelineRolePermissionBoundary", "ParameterValue": puppet_code_pipeline_role_permission_boundary, "UsePreviousValue": False, }, { "ParameterKey": "SourceRolePermissionsBoundary", "ParameterValue": source_role_permissions_boundary, "UsePreviousValue": False, }, { "ParameterKey": "PuppetGenerateRolePermissionBoundary", "ParameterValue": puppet_generate_role_permission_boundary, "UsePreviousValue": False, }, { "ParameterKey": "PuppetDeployRolePermissionBoundary", "ParameterValue": puppet_deploy_role_permission_boundary, "UsePreviousValue": False, }, { "ParameterKey": "PuppetProvisioningRolePermissionsBoundary", "ParameterValue": puppet_provisioning_role_permissions_boundary, "UsePreviousValue": False, }, { "ParameterKey": "CloudFormationDeployRolePermissionsBoundary", "ParameterValue": cloud_formation_deploy_role_permissions_boundary, "UsePreviousValue": False, }, { "ParameterKey": "DeployEnvironmentComputeType", "ParameterValue": deploy_environment_compute_type, "UsePreviousValue": False, }, { "ParameterKey": "SpokeDeployEnvironmentComputeType", "ParameterValue": spoke_deploy_environment_compute_type, "UsePreviousValue": False, }, { "ParameterKey": "DeployNumWorkers", "ParameterValue": str(deploy_num_workers), "UsePreviousValue": False, }, { "ParameterKey": "PuppetRoleName", "ParameterValue": puppet_role_name, "UsePreviousValue": False, }, { "ParameterKey": "PuppetRolePath", "ParameterValue": puppet_role_path, "UsePreviousValue": False, }, ], "Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}] + initialiser_stack_tags, } with betterboto_client.ClientContextManager("cloudformation") as cloudformation: click.echo("Creating {}".format(constants.BOOTSTRAP_STACK_NAME)) cloudformation.create_or_update(**args) buff = io.BytesIO() with zipfile.ZipFile(buff, mode="w", compression=zipfile.ZIP_DEFLATED) as z: z.writestr("parameters.yaml", 'single_account: "000000000000"') with betterboto_client.ClientContextManager("s3") as s3: try: s3.head_object( Bucket=f"sc-puppet-parameterised-runs-{puppet_account_id}", Key="parameters.zip", ) except botocore.exceptions.ClientError as ex: if ex.response["Error"]["Code"] == "404": s3.put_object( Bucket=f"sc-puppet-parameterised-runs-{puppet_account_id}", Key="parameters.zip", Body=buff.getvalue(), ) click.echo("Finished creating {}.".format(constants.BOOTSTRAP_STACK_NAME))
def run(self): self.debug("starting") content = open(self.manifest_file_path, "r").read() new_manifest = yaml.safe_load(content) regions = config.get_regions(self.puppet_account_id) global_id_cache = dict() new_manifest["id_cache"] = global_id_cache for region in regions: regional_id_cache = dict() r = self.input().get(region) for launch_name, launch_details in self.manifest.get_launches_items( ): self.debug( f"processing launch_name={launch_name} in {region} for id_cache generation" ) target = r.get(launch_details.get("portfolio")).get("details") portfolio_id = json.loads( target.open("r").read()).get("portfolio_id") portfolio_name = launch_details.get("portfolio") if regional_id_cache.get(portfolio_name) is None: regional_id_cache[portfolio_name] = dict(id=portfolio_id, products=dict()) self.debug( f"added {portfolio_name}={portfolio_id} to id_cache") product = launch_details.get("product") target = (r.get(launch_details.get("portfolio")).get( "products").get(product)) all_details = json.loads(target.open("r").read()) all_products_and_their_versions = all_details for p in all_products_and_their_versions: product_name = p.get("Name") self.debug(f"processing product_name={product_name}") if (regional_id_cache[portfolio_name]["products"].get( product_name) is None): regional_id_cache[portfolio_name]["products"][ product_name] = dict(id=p.get("ProductId"), versions=dict()) self.debug(f"added {product_name} to id_cache") for a in p.get("provisioning_artifact_details"): version_id = a.get("Id") version_name = a.get("Name") self.debug( f"added version {version_name}={version_id} to id_cache" ) regional_id_cache[portfolio_name]["products"][ product_name]["versions"][ version_name] = version_id global_id_cache[region] = regional_id_cache bucket = f"sc-puppet-spoke-deploy-{self.puppet_account_id}" cached_output_signed_url = None if (self.input().get("parameters") or self.input().get("parameter_by_paths") or self.has_hub_and_spoke_split_execution_mode()): with zipfile.ZipFile("output/GetSSMParamTask.zip", "w", zipfile.ZIP_DEFLATED) as zip: files = list() for task_name in constants.TASKS_TO_SHARE_WITH_SPOKES: files.extend( glob.glob(f"output/{task_name}*/**", recursive=True)) for filename in files: zip.write(filename, filename) with self.hub_client("s3") as s3: key = f"{os.getenv('CODEBUILD_BUILD_NUMBER', '0')}-cached-output.zip" s3.upload_file( Filename="output/GetSSMParamTask.zip", Bucket=bucket, Key=key, ) cached_output_signed_url = s3.generate_presigned_url( "get_object", Params={ "Bucket": bucket, "Key": key }, ExpiresIn=60 * 60 * 24, ) with self.hub_client("s3") as s3: manifest_content = yaml.safe_dump(new_manifest) key = f"{os.getenv('CODEBUILD_BUILD_NUMBER', '0')}.yaml" self.debug(f"Uploading generated manifest {key} to {bucket}") s3.put_object( Body=manifest_content, Bucket=bucket, Key=key, ) self.debug(f"Generating presigned URL for {key}") signed_url = s3.generate_presigned_url( "get_object", Params={ "Bucket": bucket, "Key": key }, ExpiresIn=60 * 60 * 24, ) self.write_output( dict( manifest_content=manifest_content, signed_url=signed_url, cached_output_signed_url=cached_output_signed_url, ))
def convert_manifest_into_task_defs_for_launches( manifest, puppet_account_id, should_use_sns, should_use_product_plans, include_expanded_from=False ): task_defs = [] accounts = manifest.get('accounts', []) actions = manifest.get('actions', {}) for launch_name, launch_details in manifest.get('launches', {}).items(): logger.info(f"looking at {launch_name}") pre_actions = [] for provision_action in launch_details.get('pre_actions', []): action = deepcopy(actions.get(provision_action.get('name'))) action.update(provision_action) action['source'] = launch_name action['phase'] = 'pre' action['source_type'] = 'launch' pre_actions.append(action) post_actions = [] for provision_action in launch_details.get('post_actions', []): action = deepcopy(actions.get(provision_action.get('name'))) action.update(provision_action) action['source'] = launch_name action['phase'] = 'post' action['source_type'] = 'launch' post_actions.append(action) task_def = { 'launch_name': launch_name, 'portfolio': launch_details.get('portfolio'), 'product': launch_details.get('product'), 'version': launch_details.get('version'), 'puppet_account_id': puppet_account_id, 'parameters': [], 'ssm_param_inputs': [], 'launch_parameters': launch_details.get('parameters', {}), 'manifest_parameters': manifest.get('parameters', {}), 'depends_on': launch_details.get('depends_on', []), 'dependencies': [], 'retry_count': 0, 'worker_timeout': launch_details.get('timeoutInSeconds', constants.DEFAULT_TIMEOUT), 'ssm_param_outputs': launch_details.get('outputs', {}).get('ssm', []), 'should_use_sns': should_use_sns, 'should_use_product_plans': should_use_product_plans, 'requested_priority': 0, 'status': launch_details.get('status', constants.PROVISIONED), 'pre_actions': pre_actions, 'post_actions': post_actions, } if manifest.get('configuration'): if manifest.get('configuration').get('retry_count'): task_def['retry_count'] = manifest.get('configuration').get('retry_count') if launch_details.get('configuration'): if launch_details.get('configuration').get('retry_count'): task_def['retry_count'] = launch_details.get('configuration').get('retry_count') if launch_details.get('configuration').get('requested_priority'): task_def['requested_priority'] = int(launch_details.get('configuration').get('requested_priority')) deploy_to = launch_details .get('deploy_to') for tag_list_item in deploy_to.get('tags', []): for account in accounts: for tag in account.get('tags', []): if tag == tag_list_item.get('tag'): tag_account_def = deepcopy(task_def) tag_account_def['account_id'] = account.get('account_id') if include_expanded_from: tag_account_def['expanded_from'] = account.get('expanded_from') tag_account_def['organization'] = account.get('organization') tag_account_def['account_parameters'] = account.get('parameters', {}) regions = tag_list_item.get('regions') if isinstance(regions, str): if regions in ["enabled", "regions_enabled", "enabled_regions"]: for region_enabled in account.get('regions_enabled'): region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region_enabled task_defs.append(region_tag_account_def) elif regions == 'default_region': region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = account.get('default_region') task_defs.append(region_tag_account_def) elif regions == "all": all_regions = config.get_regions() for region_enabled in all_regions: region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region_enabled task_defs.append(region_tag_account_def) else: raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}") elif isinstance(regions, list): for region in regions: region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region task_defs.append(region_tag_account_def) else: raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}") for account_list_item in deploy_to.get('accounts', []): for account in accounts: if account.get('account_id') == account_list_item.get('account_id'): account_account_def = deepcopy(task_def) account_account_def['account_id'] = account.get('account_id') if include_expanded_from: account_account_def['expanded_from'] = account.get('expanded_from') account_account_def['organization'] = account.get('organization') account_account_def['account_parameters'] = account.get('parameters', {}) regions = account_list_item.get('regions') if isinstance(regions, str): if regions in ["enabled", "regions_enabled", "enabled_regions"]: for region_enabled in account.get('regions_enabled'): region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = region_enabled task_defs.append(region_account_account_def) elif regions in ["default_region"]: region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = account.get('default_region') task_defs.append(region_account_account_def) elif regions in ["all"]: all_regions = config.get_regions() for region_enabled in all_regions: region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = region_enabled task_defs.append(region_account_account_def) else: raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}") elif isinstance(regions, list): for region in regions: region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = region task_defs.append(region_account_account_def) else: raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}") for task_def in task_defs: for depends_on_launch_name in task_def.get('depends_on', []): for task_def_2 in task_defs: if task_def_2.get('launch_name') == depends_on_launch_name: task_def_2_copy = deepcopy(task_def_2) del task_def_2_copy['depends_on'] task_def_2_copy['dependencies'] = [] task_def['dependencies'].append(task_def_2_copy) for task_def in task_defs: del task_def['depends_on'] return task_defs
def convert_manifest_into_task_defs_for_spoke_local_portfolios(manifest, puppet_account_id, should_use_sns, launch_tasks): tasks = [] accounts = manifest.get('accounts', []) actions = manifest.get('actions', {}) for launch_name, launch_details in manifest.get('spoke-local-portfolios', {}).items(): logger.info(f"Looking at {launch_name}") pre_actions = [] for provision_action in launch_details.get('pre_actions', []): action = deepcopy(actions.get(provision_action.get('name'))) action.update(provision_action) action['source'] = launch_name action['phase'] = 'pre' action['source_type'] = 'spoke-local-portfolios' pre_actions.append(action) post_actions = [] for provision_action in launch_details.get('post_actions', []): action = deepcopy(actions.get(provision_action.get('name'))) action.update(provision_action) action['source'] = launch_name action['phase'] = 'post' action['source_type'] = 'spoke-local-portfolios' post_actions.append(action) task_def = { 'launch_tasks': launch_tasks, 'launch_details': launch_details, 'puppet_account_id': puppet_account_id, 'should_use_sns': should_use_sns, 'pre_actions': pre_actions, 'post_actions': post_actions, } if manifest.get('configuration'): if manifest.get('configuration').get('retry_count'): task_def['retry_count'] = manifest.get('configuration').get('retry_count') if launch_details.get('configuration'): if launch_details.get('configuration').get('retry_count'): task_def['retry_count'] = launch_details.get('configuration').get('retry_count') if launch_details.get('configuration').get('requested_priority'): task_def['requested_priority'] = int(launch_details.get('configuration').get('requested_priority')) deploy_to = launch_details .get('deploy_to') for tag_list_item in deploy_to.get('tags', []): for account in accounts: for tag in account.get('tags', []): if tag == tag_list_item.get('tag'): tag_account_def = deepcopy(task_def) tag_account_def['account_id'] = account.get('account_id') tag_account_def['expanded_from'] = account.get('expanded_from') tag_account_def['organization'] = account.get('organization') regions = tag_list_item.get('regions') if isinstance(regions, str): if regions in ["enabled", "regions_enabled", "enabled_regions"]: for region_enabled in account.get('regions_enabled'): region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region_enabled tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_tag_account_def ) elif regions == 'default_region': region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = account.get('default_region') tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_tag_account_def ) elif regions == "all": all_regions = config.get_regions() for region_enabled in all_regions: region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region_enabled tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_tag_account_def ) else: raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}") elif isinstance(regions, list): for region in regions: region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_tag_account_def ) else: raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}") for account_list_item in deploy_to.get('accounts', []): for account in accounts: if account.get('account_id') == account_list_item.get('account_id'): account_account_def = deepcopy(task_def) account_account_def['account_id'] = account.get('account_id') account_account_def['expanded_from'] = account.get('expanded_from') account_account_def['organization'] = account.get('organization') # account_account_def['account_parameters'] = account.get('parameters', {}) regions = account_list_item.get('regions') if isinstance(regions, str): if regions in ["enabled", "regions_enabled", "enabled_regions"]: for region_enabled in account.get('regions_enabled'): region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = region_enabled tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_account_account_def ) elif regions == 'default_region': region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = account.get('default_region') tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_account_account_def ) elif regions == "all": all_regions = config.get_regions() for region_enabled in all_regions: region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = region_enabled tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_account_account_def ) else: raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}") elif isinstance(regions, list): for region in regions: region_account_account_def = deepcopy(account_account_def) region_account_account_def['region'] = region tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in( **region_account_account_def ) else: raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}") return tasks
def expand(f, puppet_account_id, single_account, subset=None): click.echo("Expanding") manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break click.echo("Filtered") new_manifest = manifest_utils.rewrite_depends_on(new_manifest) new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest) new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id) if subset: click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate( manifest_utils.Manifest(new_manifest), subset ) new_manifest = json.loads(json.dumps(new_manifest)) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() home_region = config.get_home_region(puppet_account_id) with betterboto_client.ClientContextManager("ssm") as ssm: response = ssm.get_parameter(Name="service-catalog-puppet-version") version = response.get("Parameter").get("Value") new_manifest["config_cache"] = dict( home_region=home_region, regions=config.get_regions(puppet_account_id, home_region), should_collect_cloudformation_events=config.get_should_use_sns( puppet_account_id, home_region ), should_forward_events_to_eventbridge=config.get_should_use_eventbridge( puppet_account_id, home_region ), should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter( puppet_account_id, home_region ), puppet_version=version, ) new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def _do_bootstrap( puppet_version, puppet_account_id, with_manual_approvals, puppet_code_pipeline_role_permission_boundary, source_role_permissions_boundary, puppet_generate_role_permission_boundary, puppet_deploy_role_permission_boundary, puppet_provisioning_role_permissions_boundary, cloud_formation_deploy_role_permissions_boundary, deploy_environment_compute_type="BUILD_GENERAL1_SMALL", deploy_num_workers=10, source_provider=None, owner=None, repo=None, branch=None, poll_for_source_changes=None, webhook_secret=None, ): click.echo("Starting bootstrap") should_use_eventbridge = config.get_should_use_eventbridge( puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")) if should_use_eventbridge: with betterboto_client.ClientContextManager("events") as events: try: events.describe_event_bus(Name=constants.EVENT_BUS_NAME) except events.exceptions.ResourceNotFoundException: events.create_event_bus(Name=constants.EVENT_BUS_NAME, ) all_regions = config.get_regions(puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")) with betterboto_client.MultiRegionClientContextManager( "cloudformation", all_regions) as clients: click.echo("Creating {}-regional".format( constants.BOOTSTRAP_STACK_NAME)) threads = [] template = asset_helpers.read_from_site_packages( "{}.template.yaml".format("{}-regional".format( constants.BOOTSTRAP_STACK_NAME))) template = Template(template).render(VERSION=puppet_version) args = { "StackName": "{}-regional".format(constants.BOOTSTRAP_STACK_NAME), "TemplateBody": template, "Capabilities": ["CAPABILITY_IAM"], "Parameters": [ { "ParameterKey": "Version", "ParameterValue": puppet_version, "UsePreviousValue": False, }, { "ParameterKey": "DefaultRegionValue", "ParameterValue": os.environ.get("AWS_DEFAULT_REGION"), "UsePreviousValue": False, }, ], "Tags": [{ "Key": "ServiceCatalogPuppet:Actor", "Value": "Framework", }], } for client_region, client in clients.items(): process = Thread(name=client_region, target=client.create_or_update, kwargs=args) process.start() threads.append(process) for process in threads: process.join() click.echo("Finished creating {}-regional".format( constants.BOOTSTRAP_STACK_NAME)) source_args = {"Provider": source_provider} if source_provider == "CodeCommit": source_args.update({ "Configuration": { "RepositoryName": repo, "BranchName": branch, }, }) elif source_provider == "GitHub": source_args.update({ "Configuration": { "Owner": owner, "Repo": repo, "Branch": branch, "PollForSourceChanges": poll_for_source_changes, "SecretsManagerSecret": webhook_secret, }, }) with betterboto_client.ClientContextManager( "cloudformation") as cloudformation: click.echo("Creating {}".format(constants.BOOTSTRAP_STACK_NAME)) template = asset_helpers.read_from_site_packages( "{}.template.yaml".format(constants.BOOTSTRAP_STACK_NAME)) template = Template(template).render(VERSION=puppet_version, ALL_REGIONS=all_regions, Source=source_args) template = Template(template).render(VERSION=puppet_version, ALL_REGIONS=all_regions, Source=source_args) args = { "StackName": constants.BOOTSTRAP_STACK_NAME, "TemplateBody": template, "Capabilities": ["CAPABILITY_NAMED_IAM"], "Parameters": [ { "ParameterKey": "Version", "ParameterValue": puppet_version, "UsePreviousValue": False, }, { "ParameterKey": "OrgIamRoleArn", "ParameterValue": str(config.get_org_iam_role_arn(puppet_account_id)), "UsePreviousValue": False, }, { "ParameterKey": "WithManualApprovals", "ParameterValue": "Yes" if with_manual_approvals else "No", "UsePreviousValue": False, }, { "ParameterKey": "PuppetCodePipelineRolePermissionBoundary", "ParameterValue": puppet_code_pipeline_role_permission_boundary, "UsePreviousValue": False, }, { "ParameterKey": "SourceRolePermissionsBoundary", "ParameterValue": source_role_permissions_boundary, "UsePreviousValue": False, }, { "ParameterKey": "PuppetGenerateRolePermissionBoundary", "ParameterValue": puppet_generate_role_permission_boundary, "UsePreviousValue": False, }, { "ParameterKey": "PuppetDeployRolePermissionBoundary", "ParameterValue": puppet_deploy_role_permission_boundary, "UsePreviousValue": False, }, { "ParameterKey": "PuppetProvisioningRolePermissionsBoundary", "ParameterValue": puppet_provisioning_role_permissions_boundary, "UsePreviousValue": False, }, { "ParameterKey": "CloudFormationDeployRolePermissionsBoundary", "ParameterValue": cloud_formation_deploy_role_permissions_boundary, "UsePreviousValue": False, }, { "ParameterKey": "DeployEnvironmentComputeType", "ParameterValue": deploy_environment_compute_type, "UsePreviousValue": False, }, { "ParameterKey": "DeployNumWorkers", "ParameterValue": str(deploy_num_workers), "UsePreviousValue": False, }, ], } cloudformation.create_or_update(**args) click.echo("Finished creating {}.".format(constants.BOOTSTRAP_STACK_NAME)) if source_provider == "CodeCommit": with betterboto_client.ClientContextManager( "codecommit") as codecommit: response = codecommit.get_repository(repositoryName=repo) clone_url = response.get("repositoryMetadata").get("cloneUrlHttp") clone_command = ( "git clone --config 'credential.helper=!aws codecommit credential-helper $@' " "--config 'credential.UseHttpPath=true' {}".format(clone_url)) click.echo( "You need to clone your newly created repo now and will then need to seed it: \n{}" .format(clone_command))
def expand(f, puppet_account_id, single_account, subset=None): click.echo("Expanding") target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"]) assemble_manifest_from_ssm(target_directory) manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios( new_manifest ) if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break items_to_delete = list() for section_name in constants.ALL_SECTION_NAMES: deploy_to_name = constants.DEPLOY_TO_NAMES[section_name] for item_name, item in new_manifest.get(section_name, {}).items(): accounts = list() for deploy_details in item.get(deploy_to_name, {}).get("accounts", []): if str(deploy_details.get("account_id")) == str(single_account): accounts.append(deploy_details) print(f"{item_name}: there are " + str(len(accounts))) if item.get(deploy_to_name).get("accounts"): if len(accounts) > 0: item[deploy_to_name]["accounts"] = accounts else: if item[deploy_to_name].get("tags") or item[deploy_to_name].get( "ous" ): del item[deploy_to_name]["accounts"] else: items_to_delete.append(f"{section_name}:{item_name}") for item_to_delete in items_to_delete: section_name, item_name = item_to_delete.split(":") del new_manifest[section_name][item_name] click.echo("Filtered") new_manifest = manifest_utils.rewrite_cfct(new_manifest) new_manifest = manifest_utils.rewrite_depends_on(new_manifest) new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest) new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id) new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id) new_manifest = manifest_utils.parse_conditions(new_manifest) if subset and subset.get("section"): click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate(new_manifest, subset) manifest_accounts_all = [ {"account_id": a.get("account_id"), "email": a.get("email")} for a in new_manifest.get("accounts", []) ] manifest_accounts_excluding = [ a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id ] # handle all accounts sct_manifest_accounts = json.dumps(manifest_accounts_all) sct_manifest_spokes = json.dumps(manifest_accounts_excluding) regions = config.get_regions(puppet_account_id) sct_config_regions = json.dumps(regions) new_manifest["parameters"]["SCTManifestAccounts"] = dict( default=sct_manifest_accounts ) new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes) new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions) new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() home_region = config.get_home_region(puppet_account_id) with betterboto_client.ClientContextManager("ssm") as ssm: response = ssm.get_parameter(Name="service-catalog-puppet-version") version = response.get("Parameter").get("Value") new_manifest["config_cache"] = dict( home_region=home_region, regions=regions, should_collect_cloudformation_events=config.get_should_use_sns( puppet_account_id, home_region ), should_forward_events_to_eventbridge=config.get_should_use_eventbridge( puppet_account_id, home_region ), should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter( puppet_account_id, home_region ), puppet_version=version, ) new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml_utils.dump(new_manifest))
def get_task_defs_from_details(launch_details, accounts, include_expanded_from, launch_name, configuration): tasks = [] deploy_to = launch_details.get('deploy_to') for tag_list_item in deploy_to.get('tags', []): for account in accounts: for tag in account.get('tags', []): if tag == tag_list_item.get('tag'): tag_account_def = deepcopy(configuration) tag_account_def['account_id'] = account.get('account_id') tag_account_def['expanded_from'] = account.get( 'expanded_from', '') tag_account_def['organization'] = account.get( 'organization') regions = tag_list_item.get('regions', 'default_region') if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions" ]: for region_enabled in account.get( 'regions_enabled'): region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def[ 'region'] = region_enabled tasks.append(region_tag_account_def) elif regions == 'default_region': region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = account.get( 'default_region') tasks.append(region_tag_account_def) elif regions == "all": all_regions = config.get_regions() for region_enabled in all_regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def[ 'region'] = region_enabled tasks.append(region_tag_account_def) else: raise Exception( f"Unsupported regions {regions} setting for launch: {launch_name}" ) elif isinstance(regions, list): for region in regions: region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region tasks.append(region_tag_account_def) elif isinstance(regions, tuple): for region in regions: region_tag_account_def = deepcopy(tag_account_def) region_tag_account_def['region'] = region tasks.append(region_tag_account_def) else: raise Exception( f"Unexpected regions of {regions} set for launch {launch_name}" ) for account_list_item in deploy_to.get('accounts', []): for account in accounts: if account.get('account_id') == account_list_item.get( 'account_id'): account_account_def = deepcopy(configuration) account_account_def['account_id'] = account.get('account_id') account_account_def['expanded_from'] = account.get( 'expanded_from') account_account_def['organization'] = account.get( 'organization') # account_account_def['account_parameters'] = account.get('parameters', {}) regions = account_list_item.get('regions', 'default_region') if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions" ]: for region_enabled in account.get('regions_enabled'): region_account_account_def = deepcopy( account_account_def) region_account_account_def[ 'region'] = region_enabled tasks.append(region_account_account_def) elif regions == 'default_region': region_account_account_def = deepcopy( account_account_def) region_account_account_def['region'] = account.get( 'default_region') tasks.append(region_account_account_def) elif regions == "all": all_regions = config.get_regions() for region_enabled in all_regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def[ 'region'] = region_enabled tasks.append(region_account_account_def) else: raise Exception( f"Unsupported regions {regions} setting for launch: {launch_name}" ) elif isinstance(regions, list): for region in regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def['region'] = region tasks.append(region_account_account_def) elif isinstance(regions, tuple): for region in regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def['region'] = region tasks.append(region_account_account_def) else: raise Exception( f"Unexpected regions of {regions} set for launch {launch_name}" ) return tasks
def run_tasks_for_generate_shares(tasks_to_run): for type in [ "failure", "success", "timeout", "process_failure", "processing_time", "broken_task", ]: os.makedirs(Path(constants.RESULTS_DIRECTORY) / type) run_result = luigi.build( tasks_to_run, local_scheduler=True, detailed_summary=True, workers=10, log_level='INFO', ) should_use_sns = config.get_should_use_sns() puppet_account_id = config.get_puppet_account_id() version = config.get_puppet_version() for region in config.get_regions(): sharing_policies = { 'accounts': [], 'organizations': [], } with betterboto_client.ClientContextManager( 'cloudformation', region_name=region) as cloudformation: cloudformation.ensure_deleted( StackName="servicecatalog-puppet-shares") logger.info(f"generating policies collection for region {region}") if os.path.exists(os.path.sep.join(['data', 'bucket'])): logger.info(f"Updating policies for the region: {region}") path = os.path.sep.join(['data', 'bucket', region, 'accounts']) if os.path.exists(path): for account_file in os.listdir(path): account = account_file.split(".")[0] sharing_policies['accounts'].append(account) path = os.path.sep.join( ['data', 'bucket', region, 'organizations']) if os.path.exists(path): for organization_file in os.listdir(path): organization = organization_file.split(".")[0] sharing_policies['organizations'].append(organization) logger.info(f"Finished generating policies collection") template = config.env.get_template( 'policies.template.yaml.j2').render( sharing_policies=sharing_policies, VERSION=version, ) with betterboto_client.ClientContextManager( 'cloudformation', region_name=region) as cloudformation: cloudformation.create_or_update( StackName="servicecatalog-puppet-policies", TemplateBody=template, NotificationARNs=[ f"arn:aws:sns:{region}:{puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events" ] if should_use_sns else [], ) for filename in glob('results/failure/*.json'): result = json.loads(open(filename, 'r').read()) click.echo( colorclass.Color("{red}" + result.get('task_type') + " failed{/red}")) click.echo( f"{yaml.safe_dump({'parameters':result.get('task_params')})}") click.echo("\n".join(result.get('exception_stack_trace'))) click.echo('') exit_status_codes = { LuigiStatusCode.SUCCESS: 0, LuigiStatusCode.SUCCESS_WITH_RETRY: 0, LuigiStatusCode.FAILED: 1, LuigiStatusCode.FAILED_AND_SCHEDULING_FAILED: 2, LuigiStatusCode.SCHEDULING_FAILED: 3, LuigiStatusCode.NOT_RUN: 4, LuigiStatusCode.MISSING_EXT: 5, } sys.exit(exit_status_codes.get(run_result.status))
def get_task_defs_from_details( self, puppet_account_id, launch_name, configuration, launch_or_spoke_local_portfolio, ): launch_details = self.get(launch_or_spoke_local_portfolio).get( launch_name) accounts = self.get("accounts") if launch_details is None: raise Exception(f"launch_details is None for {launch_name}") if launch_or_spoke_local_portfolio == "lambda-invocations": deploy_to = launch_details.get("invoke_for") elif launch_or_spoke_local_portfolio == "launches": deploy_to = launch_details.get("deploy_to") elif launch_or_spoke_local_portfolio == "spoke-local-portfolios": deploy_to = launch_details.get("deploy_to") or launch_details.get( "share_with") task_defs = [] for tag_list_item in deploy_to.get("tags", []): for account in accounts: for tag in account.get("tags", []): if tag == tag_list_item.get("tag"): tag_account_def = deepcopy(configuration) tag_account_def["account_id"] = account.get( "account_id") tag_account_def["account_parameters"] = account.get( "parameters", {}) regions = tag_list_item.get("regions", "default_region") if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions", ]: for region_enabled in account.get( "regions_enabled"): region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def[ "region"] = region_enabled task_defs.append(region_tag_account_def) elif regions == "default_region": region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def["region"] = account.get( "default_region") task_defs.append(region_tag_account_def) elif regions == "all": all_regions = config.get_regions( puppet_account_id) for region_enabled in all_regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def[ "region"] = region_enabled task_defs.append(region_tag_account_def) else: raise Exception( f"Unsupported regions {regions} setting for launch: {launch_name}" ) elif isinstance(regions, list): for region in regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def["region"] = region task_defs.append(region_tag_account_def) elif isinstance(regions, tuple): for region in regions: region_tag_account_def = deepcopy( tag_account_def) region_tag_account_def["region"] = region task_defs.append(region_tag_account_def) else: raise Exception( f"Unexpected regions of {regions} set for launch {launch_name}" ) for account_list_item in deploy_to.get("accounts", []): for account in accounts: if account.get("account_id") == account_list_item.get( "account_id"): account_account_def = deepcopy(configuration) account_account_def["account_id"] = account.get( "account_id") account_account_def["account_parameters"] = account.get( "parameters", {}) regions = account_list_item.get("regions", "default_region") if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions" ]: for region_enabled in account.get( "regions_enabled"): region_account_account_def = deepcopy( account_account_def) region_account_account_def[ "region"] = region_enabled task_defs.append(region_account_account_def) elif regions in ["default_region"]: region_account_account_def = deepcopy( account_account_def) region_account_account_def["region"] = account.get( "default_region") task_defs.append(region_account_account_def) elif regions in ["all"]: all_regions = config.get_regions(puppet_account_id) for region_enabled in all_regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def[ "region"] = region_enabled task_defs.append(region_account_account_def) else: raise Exception( f"Unsupported regions {regions} setting for launch: {launch_name}" ) elif isinstance(regions, list): for region in regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def["region"] = region task_defs.append(region_account_account_def) elif isinstance(regions, tuple): for region in regions: region_account_account_def = deepcopy( account_account_def) region_account_account_def["region"] = region task_defs.append(region_account_account_def) else: raise Exception( f"Unexpected regions of {regions} set for launch {launch_name}" ) return task_defs
def get_tasks_for(self, puppet_account_id, section_name, item_name, single_account="None"): accounts = self.get(constants.ACCOUNTS) section = self.get(section_name) provisioning_tasks = list() item = section[item_name] deploy_to = { "launches": "deploy_to", "stacks": "deploy_to", "apps": "deploy_to", "workspaces": "deploy_to", "spoke-local-portfolios": "share_with", "lambda-invocations": "invoke_for", "code-build-runs": "run_for", "assertions": "assert_for", }.get(section_name) if (section_name == constants.SPOKE_LOCAL_PORTFOLIOS and item.get(deploy_to) is None): deploy_to = "deploy_to" common_parameters = { "launches": dict( puppet_account_id=puppet_account_id, launch_name=item_name, launch_parameters=item.get("parameters", {}), manifest_parameters=self.get("parameters", {}), ssm_param_outputs=item.get("outputs", {}).get("ssm", []), portfolio=item.get("portfolio"), product=item.get("product"), version=item.get("version"), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), requested_priority=item.get("requested_priority", 0), ), "stacks": dict( puppet_account_id=puppet_account_id, stack_name=item_name, launch_name=item.get("launch_name", ""), stack_set_name=item.get("stack_set_name", ""), launch_parameters=item.get("parameters", {}), capabilities=item.get("capabilities", []), manifest_parameters=self.get("parameters", {}), ssm_param_outputs=item.get("outputs", {}).get("ssm", []), bucket=f"sc-puppet-stacks-repository-{puppet_account_id}", key=item.get("key"), version_id=item.get("version_id", ""), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), use_service_role=item.get( constants.MANIFEST_SHOULD_USE_STACKS_SERVICE_ROLE, constants.CONFIG_SHOULD_USE_STACKS_SERVICE_ROLE_DEFAULT, ), requested_priority=item.get("requested_priority", 0), ), "apps": dict( puppet_account_id=puppet_account_id, app_name=item_name, launch_parameters=item.get("parameters", {}), manifest_parameters=self.get("parameters", {}), ssm_param_outputs=item.get("outputs", {}).get("ssm", []), bucket=f"sc-puppet-stacks-repository-{puppet_account_id}", key=item.get("key"), version_id=item.get("version_id", ""), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), requested_priority=item.get("requested_priority", 0), ), "workspaces": dict( puppet_account_id=puppet_account_id, workspace_name=item_name, launch_parameters=item.get("parameters", {}), manifest_parameters=self.get("parameters", {}), ssm_param_outputs=item.get("outputs", {}).get("ssm", []), bucket=f"sc-puppet-stacks-repository-{puppet_account_id}", key=item.get("key"), version_id=item.get("version_id", ""), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), requested_priority=item.get("requested_priority", 0), ), "spoke-local-portfolios": dict( puppet_account_id=puppet_account_id, spoke_local_portfolio_name=item_name, product_generation_method=item.get( "product_generation_method", constants.PRODUCT_GENERATION_METHOD_DEFAULT, ), organization=item.get("organization", ""), sharing_mode=item.get("sharing_mode", constants.SHARING_MODE_DEFAULT), associations=item.get("associations", list()), launch_constraints=item.get("constraints", {}).get("launch", []), portfolio=item.get("portfolio"), ), "lambda-invocations": dict( puppet_account_id=puppet_account_id, lambda_invocation_name=item_name, function_name=item.get("function_name"), qualifier=item.get("qualifier", "$LATEST"), invocation_type=item.get("invocation_type"), launch_parameters=item.get("parameters", {}), manifest_parameters=self.get("parameters", {}), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), ), "code-build-runs": dict( puppet_account_id=puppet_account_id, code_build_run_name=item_name, launch_parameters=item.get("parameters", {}), manifest_parameters=self.get("parameters", {}), project_name=item.get("project_name"), requested_priority=item.get("requested_priority", 0), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), ), "assertions": dict( puppet_account_id=puppet_account_id, requested_priority=item.get("requested_priority", 0), assertion_name=item_name, expected=item.get("expected"), actual=item.get("actual"), execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT), ), }.get(section_name) tags = item.get(deploy_to).get("tags", []) for tag in tags: tag_name = tag.get("tag") regions = tag.get("regions") for account in accounts: account_id = str(account.get("account_id")) if single_account != "None" and single_account != account_id: continue additional_parameters = { "launches": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "apps": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "workspaces": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "stacks": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "spoke-local-portfolios": dict(account_id=account_id, ), "assertions": dict(account_id=account_id, ), "lambda-invocations": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "code-build-runs": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), }.get(section_name) if tag_name in account.get("tags"): if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions", ]: for region_enabled in account.get( "regions_enabled"): provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_enabled, )) elif regions == "default_region": provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=account.get("default_region"), )) elif regions == "all": all_regions = config.get_regions(puppet_account_id) for region_enabled in all_regions: provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_enabled, )) else: raise Exception( f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}" ) elif isinstance(regions, list): for region_ in regions: provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_, )) elif isinstance(regions, tuple): for region_ in regions: provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_, )) else: raise Exception( f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}" ) for account_to_deploy_to in item.get(deploy_to).get("accounts", []): account_id_of_account_to_deploy_to = account_to_deploy_to.get( "account_id") regions = account_to_deploy_to.get("regions") account = self.get_account(account_id_of_account_to_deploy_to) account_id = account_id_of_account_to_deploy_to if single_account != "None" and single_account != account_id: continue additional_parameters = { "launches": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "stacks": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "spoke-local-portfolios": dict(account_id=account_id, ), "assertions": dict(account_id=account_id, ), "lambda-invocations": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), "code-build-runs": dict( account_id=account_id, account_parameters=account.get("parameters", {}), ), }.get(section_name) if isinstance(regions, str): if regions in [ "enabled", "regions_enabled", "enabled_regions", ]: for region_enabled in account.get("regions_enabled"): provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_enabled, )) elif regions == "default_region": provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=account.get("default_region"), )) elif regions == "all": all_regions = config.get_regions(puppet_account_id) for region_enabled in all_regions: provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_enabled, )) else: raise Exception( f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}" ) elif isinstance(regions, list): for region_ in regions: provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_, )) elif isinstance(regions, tuple): for region_ in regions: provisioning_tasks.append( dict( **common_parameters, **additional_parameters, region=region_, )) else: raise Exception( f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}" ) return provisioning_tasks