def _do_bootstrap(puppet_version, with_manual_approvals):
click.echo('Starting bootstrap')
should_use_eventbridge = config.get_should_use_eventbridge(
os.environ.get("AWS_DEFAULT_REGION"))
if should_use_eventbridge:
with betterboto_client.ClientContextManager('events') as events:
try:
events.describe_event_bus(Name=constants.EVENT_BUS_NAME)
except events.exceptions.ResourceNotFoundException:
events.create_event_bus(Name=constants.EVENT_BUS_NAME, )
all_regions = config.get_regions(os.environ.get("AWS_DEFAULT_REGION"))
with betterboto_client.MultiRegionClientContextManager(
'cloudformation', all_regions) as clients:
click.echo('Creating {}-regional'.format(
constants.BOOTSTRAP_STACK_NAME))
threads = []
template = asset_helpers.read_from_site_packages(
'{}.template.yaml'.format('{}-regional'.format(
constants.BOOTSTRAP_STACK_NAME)))
template = Template(template).render(VERSION=puppet_version)
args = {
'StackName':
'{}-regional'.format(constants.BOOTSTRAP_STACK_NAME),
'TemplateBody':
template,
'Capabilities': ['CAPABILITY_IAM'],
'Parameters': [
{
'ParameterKey': 'Version',
'ParameterValue': puppet_version,
'UsePreviousValue': False,
},
{
'ParameterKey': 'DefaultRegionValue',
'ParameterValue': os.environ.get('AWS_DEFAULT_REGION'),
'UsePreviousValue': False,
},
],
'Tags': [{
"Key": "ServiceCatalogPuppet:Actor",
"Value": "Framework",
}]
}
for client_region, client in clients.items():
process = Thread(name=client_region,
target=client.create_or_update,
kwargs=args)
process.start()
threads.append(process)
for process in threads:
process.join()
click.echo('Finished creating {}-regional'.format(
constants.BOOTSTRAP_STACK_NAME))
with betterboto_client.ClientContextManager(
'cloudformation') as cloudformation:
click.echo('Creating {}'.format(constants.BOOTSTRAP_STACK_NAME))
template = asset_helpers.read_from_site_packages(
'{}.template.yaml'.format(constants.BOOTSTRAP_STACK_NAME))
template = Template(template).render(VERSION=puppet_version,
ALL_REGIONS=all_regions)
args = {
'StackName':
constants.BOOTSTRAP_STACK_NAME,
'TemplateBody':
template,
'Capabilities': ['CAPABILITY_NAMED_IAM'],
'Parameters': [
{
'ParameterKey': 'Version',
'ParameterValue': puppet_version,
'UsePreviousValue': False,
},
{
'ParameterKey': 'OrgIamRoleArn',
'ParameterValue': str(config.get_org_iam_role_arn()),
'UsePreviousValue': False,
},
{
'ParameterKey': 'WithManualApprovals',
'ParameterValue': "Yes" if with_manual_approvals else "No",
'UsePreviousValue': False,
},
],
}
cloudformation.create_or_update(**args)
click.echo('Finished creating {}.'.format(constants.BOOTSTRAP_STACK_NAME))
with betterboto_client.ClientContextManager('codecommit') as codecommit:
response = codecommit.get_repository(
repositoryName=constants.SERVICE_CATALOG_PUPPET_REPO_NAME)
clone_url = response.get('repositoryMetadata').get('cloneUrlHttp')
clone_command = "git clone --config 'credential.helper=!aws codecommit credential-helper $@' " \
"--config 'credential.UseHttpPath=true' {}".format(clone_url)
click.echo(
'You need to clone your newly created repo now and will then need to seed it: \n{}'
.format(clone_command))
def list_launches(f, format):
manifest = manifest_utils.load(f)
if format == "table":
click.echo("Getting details from your account...")
all_regions = config.get_regions(os.environ.get("AWS_DEFAULT_REGION"))
account_ids = [a.get('account_id') for a in manifest.get('accounts')]
deployments = {}
for account_id in account_ids:
for region_name in all_regions:
role = "arn:aws:iam::{}:role/{}".format(
account_id, 'servicecatalog-puppet/PuppetRole')
logger.info("Looking at region: {} in account: {}".format(
region_name, account_id))
with betterboto_client.CrossAccountClientContextManager(
'servicecatalog',
role,
'sc-{}-{}'.format(account_id, region_name),
region_name=region_name) as spoke_service_catalog:
response = spoke_service_catalog.list_accepted_portfolio_shares(
)
portfolios = response.get('PortfolioDetails', [])
response = spoke_service_catalog.list_portfolios()
portfolios += response.get('PortfolioDetails', [])
for portfolio in portfolios:
portfolio_id = portfolio.get('Id')
response = spoke_service_catalog.search_products_as_admin(
PortfolioId=portfolio_id)
for product_view_detail in response.get(
'ProductViewDetails', []):
product_view_summary = product_view_detail.get(
'ProductViewSummary')
product_id = product_view_summary.get('ProductId')
response = spoke_service_catalog.search_provisioned_products(
Filters={
'SearchQuery':
["productId:{}".format(product_id)]
})
for provisioned_product in response.get(
'ProvisionedProducts', []):
launch_name = provisioned_product.get('Name')
status = provisioned_product.get('Status')
provisioning_artifact_response = spoke_service_catalog.describe_provisioning_artifact(
ProvisioningArtifactId=provisioned_product.get(
'ProvisioningArtifactId'),
ProductId=provisioned_product.get('ProductId'),
).get('ProvisioningArtifactDetail')
if deployments.get(account_id) is None:
deployments[account_id] = {
'account_id': account_id,
constants.LAUNCHES: {}
}
if deployments[account_id][constants.LAUNCHES].get(
launch_name) is None:
deployments[account_id][
constants.LAUNCHES][launch_name] = {}
deployments[account_id][constants.LAUNCHES][
launch_name][region_name] = {
'launch_name':
launch_name,
'portfolio':
portfolio.get('DisplayName'),
'product':
manifest.get(constants.LAUNCHES,
{}).get(launch_name,
{}).get('product'),
'version':
provisioning_artifact_response.get('Name'),
'active':
provisioning_artifact_response.get(
'Active'),
'region':
region_name,
'status':
status,
}
output_path = os.path.sep.join([
constants.LAUNCHES_PATH,
account_id,
region_name,
])
if not os.path.exists(output_path):
os.makedirs(output_path)
output = os.path.sep.join([
output_path,
"{}.json".format(provisioned_product.get('Id'))
])
with open(output, 'w') as f:
f.write(
json.dumps(provisioned_product,
indent=4,
default=str))
results = {}
tasks = generate_tasks(f)
# deployments[account_id][constants.LAUNCHES][launch_name][region_name]
for task in tasks:
account_id = task.get('account_id')
launch_name = task.get('launch_name')
if deployments.get(account_id, {}).get(constants.LAUNCHES,
{}).get(launch_name) is None:
pass
else:
for region, regional_details in deployments[account_id][
constants.LAUNCHES][launch_name].items():
results[f"{account_id}_{region}_{launch_name}"] = {
'account_id': account_id,
'region': region,
'launch': launch_name,
'portfolio': regional_details.get('portfolio'),
'product': regional_details.get('product'),
'expected_version': task.get('version'),
'actual_version': regional_details.get('version'),
'active': regional_details.get('active'),
'status': regional_details.get('status'),
}
if format == "table":
table = [[
'account_id',
'region',
'launch',
'portfolio',
'product',
'expected_version',
'actual_version',
'active',
'status',
]]
for result in results.values():
table.append([
result.get('account_id'),
result.get('region'),
result.get('launch'),
result.get('portfolio'),
result.get('product'),
result.get('expected_version'),
Color("{green}" + result.get('actual_version') + "{/green}") if
result.get('actual_version') == result.get('expected_version')
else Color("{red}" + result.get('actual_version') + "{/red}"),
Color("{green}" + str(result.get('active')) +
"{/green}") if result.get('active') else
Color("{red}" + str(result.get('active')) + "{/red}"),
Color("{green}" + result.get('status') +
"{/green}") if result.get('status') == "AVAILABLE" else
Color("{red}" + result.get('status') + "{/red}")
])
click.echo(terminaltables.AsciiTable(table).table)
elif format == "json":
click.echo(json.dumps(
results,
indent=4,
default=str,
))
else:
raise Exception(f"Unsupported format: {format}")
def requires(self):
requirements = dict()
regions = config.get_regions(self.puppet_account_id)
for launch_name, launch_details in self.manifest.get_launches_items():
portfolio = launch_details.get("portfolio")
for region in regions:
if requirements.get(region) is None:
requirements[region] = dict()
regional_details = requirements[region]
if regional_details.get(portfolio) is None:
regional_details[portfolio] = dict(products=dict())
portfolio_details = regional_details[portfolio]
if portfolio_details.get("details") is None:
portfolio_details[
"details"] = get_portfolio_by_portfolio_name_task.GetPortfolioByPortfolioName(
manifest_file_path=self.manifest_file_path,
portfolio=portfolio,
puppet_account_id=self.puppet_account_id,
account_id=self.puppet_account_id,
region=region,
)
product = launch_details.get("product")
products = portfolio_details.get("products")
if products.get(product) is None:
products[
product] = get_products_and_provisioning_artifacts_task.GetProductsAndProvisioningArtifactsTask(
manifest_file_path=self.manifest_file_path,
region=region,
portfolio=portfolio,
puppet_account_id=self.puppet_account_id,
)
params = dict()
parameter_by_paths = dict()
requirements["parameters"] = params
requirements["parameter_by_paths"] = parameter_by_paths
home_region = config.get_home_region(self.puppet_account_id)
for section in constants.SECTION_NAMES_THAT_SUPPORTS_PARAMETERS:
for item_name, item_details in self.manifest.get(section,
{}).items():
if item_details.get(
"execution") == constants.EXECUTION_MODE_SPOKE:
for parameter_name, parameter_details in item_details.get(
"parameters", {}).items():
if parameter_details.get("ssm") and str(
parameter_details.get("ssm").get(
"account_id", "")) == str(
self.puppet_account_id):
r = parameter_details.get("ssm").get(
"region",
config.get_home_region(self.puppet_account_id))
name = parameter_details.get("ssm").get("name")
path = parameter_details.get("ssm").get("path", "")
if path == "":
accounts_and_regions = self.manifest.get_account_ids_and_regions_used_for_section_item(
self.puppet_account_id, section, item_name)
for account_id, regions in accounts_and_regions.items(
):
for region in regions:
n = name.replace(
"${AWS::AccountId}",
account_id).replace(
"${AWS::Region}", region)
params[
f"{parameter_name}||{n}||{r}"] = get_ssm_param_task.GetSSMParamTask(
parameter_name=parameter_name,
name=n,
region=r,
default_value=parameter_details
.get("ssm").get(
"default_value"),
path=parameter_details.get(
"ssm").get("path", ""),
recursive=parameter_details.
get("ssm").get(
"recursive", True),
depends_on=parameter_details.
get("ssm").get(
"depends_on", []),
manifest_file_path=self.
manifest_file_path,
puppet_account_id=self.
puppet_account_id,
spoke_account_id=self.
puppet_account_id,
spoke_region=r,
)
else:
parameter_by_paths[
path] = get_ssm_param_task.GetSSMParamByPathTask(
path=parameter_details.get("ssm").get(
"path", ""),
recursive=parameter_details.get(
"ssm").get("recursive", True),
region=parameter_details.get(
"ssm").get("recursive",
home_region),
depends_on=parameter_details.get(
"ssm").get("depends_on", []),
manifest_file_path=self.
manifest_file_path,
puppet_account_id=self.
puppet_account_id,
spoke_account_id=self.
puppet_account_id,
spoke_region=home_region,
)
return requirements
def get_task_defs_from_details(
self,
puppet_account_id,
include_expanded_from,
launch_name,
configuration,
launch_or_spoke_local_portfolio,
):
logger.info(
f"get_task_defs_from_details({include_expanded_from}, {launch_name}, {configuration})"
)
launch_details = self.get(launch_or_spoke_local_portfolio).get(
launch_name)
logger.info(launch_details)
accounts = self.get("accounts")
if launch_details is None:
raise Exception(f"launch_details is None for {launch_name}")
deploy_to = launch_details.get("deploy_to")
task_defs = []
for tag_list_item in deploy_to.get("tags", []):
for account in accounts:
for tag in account.get("tags", []):
if tag == tag_list_item.get("tag"):
tag_account_def = deepcopy(configuration)
tag_account_def["account_id"] = account.get(
"account_id")
if include_expanded_from:
tag_account_def["expanded_from"] = account.get(
"expanded_from")
tag_account_def["organization"] = account.get(
"organization")
tag_account_def["account_parameters"] = account.get(
"parameters", {})
regions = tag_list_item.get("regions",
"default_region")
if isinstance(regions, str):
if regions in [
"enabled",
"regions_enabled",
"enabled_regions",
]:
for region_enabled in account.get(
"regions_enabled"):
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def[
"region"] = region_enabled
task_defs.append(region_tag_account_def)
elif regions == "default_region":
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def["region"] = account.get(
"default_region")
task_defs.append(region_tag_account_def)
elif regions == "all":
all_regions = config.get_regions(
puppet_account_id)
for region_enabled in all_regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def[
"region"] = region_enabled
task_defs.append(region_tag_account_def)
else:
raise Exception(
f"Unsupported regions {regions} setting for launch: {launch_name}"
)
elif isinstance(regions, list):
for region in regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def["region"] = region
task_defs.append(region_tag_account_def)
elif isinstance(regions, tuple):
for region in regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def["region"] = region
task_defs.append(region_tag_account_def)
else:
raise Exception(
f"Unexpected regions of {regions} set for launch {launch_name}"
)
for account_list_item in deploy_to.get("accounts", []):
for account in accounts:
if account.get("account_id") == account_list_item.get(
"account_id"):
account_account_def = deepcopy(configuration)
account_account_def["account_id"] = account.get(
"account_id")
if include_expanded_from:
account_account_def["expanded_from"] = account.get(
"expanded_from")
account_account_def["organization"] = account.get(
"organization")
account_account_def["account_parameters"] = account.get(
"parameters", {})
regions = account_list_item.get("regions",
"default_region")
if isinstance(regions, str):
if regions in [
"enabled", "regions_enabled", "enabled_regions"
]:
for region_enabled in account.get(
"regions_enabled"):
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def[
"region"] = region_enabled
task_defs.append(region_account_account_def)
elif regions in ["default_region"]:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def["region"] = account.get(
"default_region")
task_defs.append(region_account_account_def)
elif regions in ["all"]:
all_regions = config.get_regions(puppet_account_id)
for region_enabled in all_regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def[
"region"] = region_enabled
task_defs.append(region_account_account_def)
else:
raise Exception(
f"Unsupported regions {regions} setting for launch: {launch_name}"
)
elif isinstance(regions, list):
for region in regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def["region"] = region
task_defs.append(region_account_account_def)
elif isinstance(regions, tuple):
for region in regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def["region"] = region
task_defs.append(region_account_account_def)
else:
raise Exception(
f"Unexpected regions of {regions} set for launch {launch_name}"
)
return task_defs
def bootstrap(
puppet_account_id,
with_manual_approvals,
puppet_code_pipeline_role_permission_boundary,
source_role_permissions_boundary,
puppet_generate_role_permission_boundary,
puppet_deploy_role_permission_boundary,
puppet_provisioning_role_permissions_boundary,
cloud_formation_deploy_role_permissions_boundary,
deploy_environment_compute_type,
spoke_deploy_environment_compute_type,
deploy_num_workers,
source_provider,
owner,
repo,
branch,
poll_for_source_changes,
webhook_secret,
puppet_role_name,
puppet_role_path,
scm_connection_arn,
scm_full_repository_id,
scm_branch_name,
scm_bucket_name,
scm_object_key,
scm_skip_creation_of_repo,
should_validate,
custom_source_action_git_url,
custom_source_action_git_web_hook_ip_address,
custom_source_action_custom_action_type_version,
custom_source_action_custom_action_type_provider,
):
click.echo("Starting bootstrap")
should_use_eventbridge = config.get_should_use_eventbridge(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
initialiser_stack_tags = config.get_initialiser_stack_tags()
if should_use_eventbridge:
with betterboto_client.ClientContextManager("events") as events:
try:
events.describe_event_bus(Name=constants.EVENT_BUS_NAME)
except events.exceptions.ResourceNotFoundException:
events.create_event_bus(Name=constants.EVENT_BUS_NAME,)
all_regions = config.get_regions(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
with betterboto_client.MultiRegionClientContextManager(
"cloudformation", all_regions
) as clients:
click.echo("Creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME))
threads = []
template = hub_bootstrap_region.get_template(
constants.VERSION, os.environ.get("AWS_DEFAULT_REGION")
).to_yaml(clean_up=True)
args = {
"StackName": "{}-regional".format(constants.BOOTSTRAP_STACK_NAME),
"TemplateBody": template,
"Capabilities": ["CAPABILITY_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": constants.VERSION,
"UsePreviousValue": False,
},
{
"ParameterKey": "DefaultRegionValue",
"ParameterValue": os.environ.get("AWS_DEFAULT_REGION"),
"UsePreviousValue": False,
},
],
"Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}]
+ initialiser_stack_tags,
}
for client_region, client in clients.items():
process = Thread(
name=client_region, target=client.create_or_update, kwargs=args
)
process.start()
threads.append(process)
for process in threads:
process.join()
click.echo(
"Finished creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME)
)
source_args = {"Provider": source_provider}
if source_provider == "CodeCommit":
source_args.update(
{
"Configuration": {
"RepositoryName": repo,
"BranchName": branch,
"PollForSourceChanges": poll_for_source_changes,
},
}
)
elif source_provider == "GitHub":
source_args.update(
{
"Configuration": {
"Owner": owner,
"Repo": repo,
"Branch": branch,
"PollForSourceChanges": poll_for_source_changes,
"SecretsManagerSecret": webhook_secret,
},
}
)
elif source_provider.lower() == "codestarsourceconnection":
source_args.update(
{
"Configuration": {
"ConnectionArn": scm_connection_arn,
"FullRepositoryId": scm_full_repository_id,
"BranchName": scm_branch_name,
"OutputArtifactFormat": "CODE_ZIP",
"DetectChanges": poll_for_source_changes,
},
}
)
elif source_provider.lower() == "s3":
source_args.update(
{
"Configuration": {
"S3Bucket": scm_bucket_name,
"S3ObjectKey": scm_object_key,
"PollForSourceChanges": poll_for_source_changes,
},
}
)
elif source_provider.lower() == "custom":
source_args.update(
{
"Configuration": {
"Owner": "Custom",
"GitUrl": custom_source_action_git_url,
"Branch": branch,
"GitWebHookIpAddress": custom_source_action_git_web_hook_ip_address,
"CustomActionTypeVersion": custom_source_action_custom_action_type_version,
"CustomActionTypeProvider": custom_source_action_custom_action_type_provider,
},
}
)
template = hub_bootstrap.get_template(
constants.VERSION,
all_regions,
source_args,
config.is_caching_enabled(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
),
with_manual_approvals,
scm_skip_creation_of_repo,
should_validate,
).to_yaml(clean_up=True)
args = {
"StackName": constants.BOOTSTRAP_STACK_NAME,
"TemplateBody": template,
"Capabilities": ["CAPABILITY_NAMED_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": constants.VERSION,
"UsePreviousValue": False,
},
{
"ParameterKey": "OrgIamRoleArn",
"ParameterValue": str(config.get_org_iam_role_arn(puppet_account_id)),
"UsePreviousValue": False,
},
{
"ParameterKey": "WithManualApprovals",
"ParameterValue": "Yes" if with_manual_approvals else "No",
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetCodePipelineRolePermissionBoundary",
"ParameterValue": puppet_code_pipeline_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "SourceRolePermissionsBoundary",
"ParameterValue": source_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetGenerateRolePermissionBoundary",
"ParameterValue": puppet_generate_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetDeployRolePermissionBoundary",
"ParameterValue": puppet_deploy_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetProvisioningRolePermissionsBoundary",
"ParameterValue": puppet_provisioning_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "CloudFormationDeployRolePermissionsBoundary",
"ParameterValue": cloud_formation_deploy_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployEnvironmentComputeType",
"ParameterValue": deploy_environment_compute_type,
"UsePreviousValue": False,
},
{
"ParameterKey": "SpokeDeployEnvironmentComputeType",
"ParameterValue": spoke_deploy_environment_compute_type,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployNumWorkers",
"ParameterValue": str(deploy_num_workers),
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetRoleName",
"ParameterValue": puppet_role_name,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetRolePath",
"ParameterValue": puppet_role_path,
"UsePreviousValue": False,
},
],
"Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}]
+ initialiser_stack_tags,
}
with betterboto_client.ClientContextManager("cloudformation") as cloudformation:
click.echo("Creating {}".format(constants.BOOTSTRAP_STACK_NAME))
cloudformation.create_or_update(**args)
buff = io.BytesIO()
with zipfile.ZipFile(buff, mode="w", compression=zipfile.ZIP_DEFLATED) as z:
z.writestr("parameters.yaml", 'single_account: "000000000000"')
with betterboto_client.ClientContextManager("s3") as s3:
try:
s3.head_object(
Bucket=f"sc-puppet-parameterised-runs-{puppet_account_id}",
Key="parameters.zip",
)
except botocore.exceptions.ClientError as ex:
if ex.response["Error"]["Code"] == "404":
s3.put_object(
Bucket=f"sc-puppet-parameterised-runs-{puppet_account_id}",
Key="parameters.zip",
Body=buff.getvalue(),
)
click.echo("Finished creating {}.".format(constants.BOOTSTRAP_STACK_NAME))
def run(self):
self.debug("starting")
content = open(self.manifest_file_path, "r").read()
new_manifest = yaml.safe_load(content)
regions = config.get_regions(self.puppet_account_id)
global_id_cache = dict()
new_manifest["id_cache"] = global_id_cache
for region in regions:
regional_id_cache = dict()
r = self.input().get(region)
for launch_name, launch_details in self.manifest.get_launches_items(
):
self.debug(
f"processing launch_name={launch_name} in {region} for id_cache generation"
)
target = r.get(launch_details.get("portfolio")).get("details")
portfolio_id = json.loads(
target.open("r").read()).get("portfolio_id")
portfolio_name = launch_details.get("portfolio")
if regional_id_cache.get(portfolio_name) is None:
regional_id_cache[portfolio_name] = dict(id=portfolio_id,
products=dict())
self.debug(
f"added {portfolio_name}={portfolio_id} to id_cache")
product = launch_details.get("product")
target = (r.get(launch_details.get("portfolio")).get(
"products").get(product))
all_details = json.loads(target.open("r").read())
all_products_and_their_versions = all_details
for p in all_products_and_their_versions:
product_name = p.get("Name")
self.debug(f"processing product_name={product_name}")
if (regional_id_cache[portfolio_name]["products"].get(
product_name) is None):
regional_id_cache[portfolio_name]["products"][
product_name] = dict(id=p.get("ProductId"),
versions=dict())
self.debug(f"added {product_name} to id_cache")
for a in p.get("provisioning_artifact_details"):
version_id = a.get("Id")
version_name = a.get("Name")
self.debug(
f"added version {version_name}={version_id} to id_cache"
)
regional_id_cache[portfolio_name]["products"][
product_name]["versions"][
version_name] = version_id
global_id_cache[region] = regional_id_cache
bucket = f"sc-puppet-spoke-deploy-{self.puppet_account_id}"
cached_output_signed_url = None
if (self.input().get("parameters")
or self.input().get("parameter_by_paths")
or self.has_hub_and_spoke_split_execution_mode()):
with zipfile.ZipFile("output/GetSSMParamTask.zip", "w",
zipfile.ZIP_DEFLATED) as zip:
files = list()
for task_name in constants.TASKS_TO_SHARE_WITH_SPOKES:
files.extend(
glob.glob(f"output/{task_name}*/**", recursive=True))
for filename in files:
zip.write(filename, filename)
with self.hub_client("s3") as s3:
key = f"{os.getenv('CODEBUILD_BUILD_NUMBER', '0')}-cached-output.zip"
s3.upload_file(
Filename="output/GetSSMParamTask.zip",
Bucket=bucket,
Key=key,
)
cached_output_signed_url = s3.generate_presigned_url(
"get_object",
Params={
"Bucket": bucket,
"Key": key
},
ExpiresIn=60 * 60 * 24,
)
with self.hub_client("s3") as s3:
manifest_content = yaml.safe_dump(new_manifest)
key = f"{os.getenv('CODEBUILD_BUILD_NUMBER', '0')}.yaml"
self.debug(f"Uploading generated manifest {key} to {bucket}")
s3.put_object(
Body=manifest_content,
Bucket=bucket,
Key=key,
)
self.debug(f"Generating presigned URL for {key}")
signed_url = s3.generate_presigned_url(
"get_object",
Params={
"Bucket": bucket,
"Key": key
},
ExpiresIn=60 * 60 * 24,
)
self.write_output(
dict(
manifest_content=manifest_content,
signed_url=signed_url,
cached_output_signed_url=cached_output_signed_url,
))
def convert_manifest_into_task_defs_for_launches(
manifest, puppet_account_id, should_use_sns, should_use_product_plans, include_expanded_from=False
):
task_defs = []
accounts = manifest.get('accounts', [])
actions = manifest.get('actions', {})
for launch_name, launch_details in manifest.get('launches', {}).items():
logger.info(f"looking at {launch_name}")
pre_actions = []
for provision_action in launch_details.get('pre_actions', []):
action = deepcopy(actions.get(provision_action.get('name')))
action.update(provision_action)
action['source'] = launch_name
action['phase'] = 'pre'
action['source_type'] = 'launch'
pre_actions.append(action)
post_actions = []
for provision_action in launch_details.get('post_actions', []):
action = deepcopy(actions.get(provision_action.get('name')))
action.update(provision_action)
action['source'] = launch_name
action['phase'] = 'post'
action['source_type'] = 'launch'
post_actions.append(action)
task_def = {
'launch_name': launch_name,
'portfolio': launch_details.get('portfolio'),
'product': launch_details.get('product'),
'version': launch_details.get('version'),
'puppet_account_id': puppet_account_id,
'parameters': [],
'ssm_param_inputs': [],
'launch_parameters': launch_details.get('parameters', {}),
'manifest_parameters': manifest.get('parameters', {}),
'depends_on': launch_details.get('depends_on', []),
'dependencies': [],
'retry_count': 0,
'worker_timeout': launch_details.get('timeoutInSeconds', constants.DEFAULT_TIMEOUT),
'ssm_param_outputs': launch_details.get('outputs', {}).get('ssm', []),
'should_use_sns': should_use_sns,
'should_use_product_plans': should_use_product_plans,
'requested_priority': 0,
'status': launch_details.get('status', constants.PROVISIONED),
'pre_actions': pre_actions,
'post_actions': post_actions,
}
if manifest.get('configuration'):
if manifest.get('configuration').get('retry_count'):
task_def['retry_count'] = manifest.get('configuration').get('retry_count')
if launch_details.get('configuration'):
if launch_details.get('configuration').get('retry_count'):
task_def['retry_count'] = launch_details.get('configuration').get('retry_count')
if launch_details.get('configuration').get('requested_priority'):
task_def['requested_priority'] = int(launch_details.get('configuration').get('requested_priority'))
deploy_to = launch_details .get('deploy_to')
for tag_list_item in deploy_to.get('tags', []):
for account in accounts:
for tag in account.get('tags', []):
if tag == tag_list_item.get('tag'):
tag_account_def = deepcopy(task_def)
tag_account_def['account_id'] = account.get('account_id')
if include_expanded_from:
tag_account_def['expanded_from'] = account.get('expanded_from')
tag_account_def['organization'] = account.get('organization')
tag_account_def['account_parameters'] = account.get('parameters', {})
regions = tag_list_item.get('regions')
if isinstance(regions, str):
if regions in ["enabled", "regions_enabled", "enabled_regions"]:
for region_enabled in account.get('regions_enabled'):
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region_enabled
task_defs.append(region_tag_account_def)
elif regions == 'default_region':
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = account.get('default_region')
task_defs.append(region_tag_account_def)
elif regions == "all":
all_regions = config.get_regions()
for region_enabled in all_regions:
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region_enabled
task_defs.append(region_tag_account_def)
else:
raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}")
elif isinstance(regions, list):
for region in regions:
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region
task_defs.append(region_tag_account_def)
else:
raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}")
for account_list_item in deploy_to.get('accounts', []):
for account in accounts:
if account.get('account_id') == account_list_item.get('account_id'):
account_account_def = deepcopy(task_def)
account_account_def['account_id'] = account.get('account_id')
if include_expanded_from:
account_account_def['expanded_from'] = account.get('expanded_from')
account_account_def['organization'] = account.get('organization')
account_account_def['account_parameters'] = account.get('parameters', {})
regions = account_list_item.get('regions')
if isinstance(regions, str):
if regions in ["enabled", "regions_enabled", "enabled_regions"]:
for region_enabled in account.get('regions_enabled'):
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = region_enabled
task_defs.append(region_account_account_def)
elif regions in ["default_region"]:
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = account.get('default_region')
task_defs.append(region_account_account_def)
elif regions in ["all"]:
all_regions = config.get_regions()
for region_enabled in all_regions:
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = region_enabled
task_defs.append(region_account_account_def)
else:
raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}")
elif isinstance(regions, list):
for region in regions:
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = region
task_defs.append(region_account_account_def)
else:
raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}")
for task_def in task_defs:
for depends_on_launch_name in task_def.get('depends_on', []):
for task_def_2 in task_defs:
if task_def_2.get('launch_name') == depends_on_launch_name:
task_def_2_copy = deepcopy(task_def_2)
del task_def_2_copy['depends_on']
task_def_2_copy['dependencies'] = []
task_def['dependencies'].append(task_def_2_copy)
for task_def in task_defs:
del task_def['depends_on']
return task_defs
def convert_manifest_into_task_defs_for_spoke_local_portfolios(manifest, puppet_account_id, should_use_sns, launch_tasks):
tasks = []
accounts = manifest.get('accounts', [])
actions = manifest.get('actions', {})
for launch_name, launch_details in manifest.get('spoke-local-portfolios', {}).items():
logger.info(f"Looking at {launch_name}")
pre_actions = []
for provision_action in launch_details.get('pre_actions', []):
action = deepcopy(actions.get(provision_action.get('name')))
action.update(provision_action)
action['source'] = launch_name
action['phase'] = 'pre'
action['source_type'] = 'spoke-local-portfolios'
pre_actions.append(action)
post_actions = []
for provision_action in launch_details.get('post_actions', []):
action = deepcopy(actions.get(provision_action.get('name')))
action.update(provision_action)
action['source'] = launch_name
action['phase'] = 'post'
action['source_type'] = 'spoke-local-portfolios'
post_actions.append(action)
task_def = {
'launch_tasks': launch_tasks,
'launch_details': launch_details,
'puppet_account_id': puppet_account_id,
'should_use_sns': should_use_sns,
'pre_actions': pre_actions,
'post_actions': post_actions,
}
if manifest.get('configuration'):
if manifest.get('configuration').get('retry_count'):
task_def['retry_count'] = manifest.get('configuration').get('retry_count')
if launch_details.get('configuration'):
if launch_details.get('configuration').get('retry_count'):
task_def['retry_count'] = launch_details.get('configuration').get('retry_count')
if launch_details.get('configuration').get('requested_priority'):
task_def['requested_priority'] = int(launch_details.get('configuration').get('requested_priority'))
deploy_to = launch_details .get('deploy_to')
for tag_list_item in deploy_to.get('tags', []):
for account in accounts:
for tag in account.get('tags', []):
if tag == tag_list_item.get('tag'):
tag_account_def = deepcopy(task_def)
tag_account_def['account_id'] = account.get('account_id')
tag_account_def['expanded_from'] = account.get('expanded_from')
tag_account_def['organization'] = account.get('organization')
regions = tag_list_item.get('regions')
if isinstance(regions, str):
if regions in ["enabled", "regions_enabled", "enabled_regions"]:
for region_enabled in account.get('regions_enabled'):
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region_enabled
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_tag_account_def
)
elif regions == 'default_region':
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = account.get('default_region')
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_tag_account_def
)
elif regions == "all":
all_regions = config.get_regions()
for region_enabled in all_regions:
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region_enabled
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_tag_account_def
)
else:
raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}")
elif isinstance(regions, list):
for region in regions:
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_tag_account_def
)
else:
raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}")
for account_list_item in deploy_to.get('accounts', []):
for account in accounts:
if account.get('account_id') == account_list_item.get('account_id'):
account_account_def = deepcopy(task_def)
account_account_def['account_id'] = account.get('account_id')
account_account_def['expanded_from'] = account.get('expanded_from')
account_account_def['organization'] = account.get('organization')
# account_account_def['account_parameters'] = account.get('parameters', {})
regions = account_list_item.get('regions')
if isinstance(regions, str):
if regions in ["enabled", "regions_enabled", "enabled_regions"]:
for region_enabled in account.get('regions_enabled'):
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = region_enabled
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_account_account_def
)
elif regions == 'default_region':
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = account.get('default_region')
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_account_account_def
)
elif regions == "all":
all_regions = config.get_regions()
for region_enabled in all_regions:
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = region_enabled
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_account_account_def
)
else:
raise Exception(f"Unsupported regions {regions} setting for launch: {launch_name}")
elif isinstance(regions, list):
for region in regions:
region_account_account_def = deepcopy(account_account_def)
region_account_account_def['region'] = region
tasks += convert_manifest_into_task_defs_for_spoke_local_portfolios_in(
**region_account_account_def
)
else:
raise Exception(f"Unexpected regions of {regions} set for launch {launch_name}")
return tasks
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
if subset:
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(
manifest_utils.Manifest(new_manifest), subset
)
new_manifest = json.loads(json.dumps(new_manifest))
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=config.get_regions(puppet_account_id, home_region),
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def _do_bootstrap(
puppet_version,
puppet_account_id,
with_manual_approvals,
puppet_code_pipeline_role_permission_boundary,
source_role_permissions_boundary,
puppet_generate_role_permission_boundary,
puppet_deploy_role_permission_boundary,
puppet_provisioning_role_permissions_boundary,
cloud_formation_deploy_role_permissions_boundary,
deploy_environment_compute_type="BUILD_GENERAL1_SMALL",
deploy_num_workers=10,
source_provider=None,
owner=None,
repo=None,
branch=None,
poll_for_source_changes=None,
webhook_secret=None,
):
click.echo("Starting bootstrap")
should_use_eventbridge = config.get_should_use_eventbridge(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION"))
if should_use_eventbridge:
with betterboto_client.ClientContextManager("events") as events:
try:
events.describe_event_bus(Name=constants.EVENT_BUS_NAME)
except events.exceptions.ResourceNotFoundException:
events.create_event_bus(Name=constants.EVENT_BUS_NAME, )
all_regions = config.get_regions(puppet_account_id,
os.environ.get("AWS_DEFAULT_REGION"))
with betterboto_client.MultiRegionClientContextManager(
"cloudformation", all_regions) as clients:
click.echo("Creating {}-regional".format(
constants.BOOTSTRAP_STACK_NAME))
threads = []
template = asset_helpers.read_from_site_packages(
"{}.template.yaml".format("{}-regional".format(
constants.BOOTSTRAP_STACK_NAME)))
template = Template(template).render(VERSION=puppet_version)
args = {
"StackName":
"{}-regional".format(constants.BOOTSTRAP_STACK_NAME),
"TemplateBody":
template,
"Capabilities": ["CAPABILITY_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": puppet_version,
"UsePreviousValue": False,
},
{
"ParameterKey": "DefaultRegionValue",
"ParameterValue": os.environ.get("AWS_DEFAULT_REGION"),
"UsePreviousValue": False,
},
],
"Tags": [{
"Key": "ServiceCatalogPuppet:Actor",
"Value": "Framework",
}],
}
for client_region, client in clients.items():
process = Thread(name=client_region,
target=client.create_or_update,
kwargs=args)
process.start()
threads.append(process)
for process in threads:
process.join()
click.echo("Finished creating {}-regional".format(
constants.BOOTSTRAP_STACK_NAME))
source_args = {"Provider": source_provider}
if source_provider == "CodeCommit":
source_args.update({
"Configuration": {
"RepositoryName": repo,
"BranchName": branch,
},
})
elif source_provider == "GitHub":
source_args.update({
"Configuration": {
"Owner": owner,
"Repo": repo,
"Branch": branch,
"PollForSourceChanges": poll_for_source_changes,
"SecretsManagerSecret": webhook_secret,
},
})
with betterboto_client.ClientContextManager(
"cloudformation") as cloudformation:
click.echo("Creating {}".format(constants.BOOTSTRAP_STACK_NAME))
template = asset_helpers.read_from_site_packages(
"{}.template.yaml".format(constants.BOOTSTRAP_STACK_NAME))
template = Template(template).render(VERSION=puppet_version,
ALL_REGIONS=all_regions,
Source=source_args)
template = Template(template).render(VERSION=puppet_version,
ALL_REGIONS=all_regions,
Source=source_args)
args = {
"StackName":
constants.BOOTSTRAP_STACK_NAME,
"TemplateBody":
template,
"Capabilities": ["CAPABILITY_NAMED_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": puppet_version,
"UsePreviousValue": False,
},
{
"ParameterKey":
"OrgIamRoleArn",
"ParameterValue":
str(config.get_org_iam_role_arn(puppet_account_id)),
"UsePreviousValue":
False,
},
{
"ParameterKey": "WithManualApprovals",
"ParameterValue": "Yes" if with_manual_approvals else "No",
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetCodePipelineRolePermissionBoundary",
"ParameterValue":
puppet_code_pipeline_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "SourceRolePermissionsBoundary",
"ParameterValue": source_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetGenerateRolePermissionBoundary",
"ParameterValue": puppet_generate_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetDeployRolePermissionBoundary",
"ParameterValue": puppet_deploy_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey":
"PuppetProvisioningRolePermissionsBoundary",
"ParameterValue":
puppet_provisioning_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey":
"CloudFormationDeployRolePermissionsBoundary",
"ParameterValue":
cloud_formation_deploy_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployEnvironmentComputeType",
"ParameterValue": deploy_environment_compute_type,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployNumWorkers",
"ParameterValue": str(deploy_num_workers),
"UsePreviousValue": False,
},
],
}
cloudformation.create_or_update(**args)
click.echo("Finished creating {}.".format(constants.BOOTSTRAP_STACK_NAME))
if source_provider == "CodeCommit":
with betterboto_client.ClientContextManager(
"codecommit") as codecommit:
response = codecommit.get_repository(repositoryName=repo)
clone_url = response.get("repositoryMetadata").get("cloneUrlHttp")
clone_command = (
"git clone --config 'credential.helper=!aws codecommit credential-helper $@' "
"--config 'credential.UseHttpPath=true' {}".format(clone_url))
click.echo(
"You need to clone your newly created repo now and will then need to seed it: \n{}"
.format(clone_command))
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"])
assemble_manifest_from_ssm(target_directory)
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios(
new_manifest
)
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
items_to_delete = list()
for section_name in constants.ALL_SECTION_NAMES:
deploy_to_name = constants.DEPLOY_TO_NAMES[section_name]
for item_name, item in new_manifest.get(section_name, {}).items():
accounts = list()
for deploy_details in item.get(deploy_to_name, {}).get("accounts", []):
if str(deploy_details.get("account_id")) == str(single_account):
accounts.append(deploy_details)
print(f"{item_name}: there are " + str(len(accounts)))
if item.get(deploy_to_name).get("accounts"):
if len(accounts) > 0:
item[deploy_to_name]["accounts"] = accounts
else:
if item[deploy_to_name].get("tags") or item[deploy_to_name].get(
"ous"
):
del item[deploy_to_name]["accounts"]
else:
items_to_delete.append(f"{section_name}:{item_name}")
for item_to_delete in items_to_delete:
section_name, item_name = item_to_delete.split(":")
del new_manifest[section_name][item_name]
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_cfct(new_manifest)
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id)
new_manifest = manifest_utils.parse_conditions(new_manifest)
if subset and subset.get("section"):
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(new_manifest, subset)
manifest_accounts_all = [
{"account_id": a.get("account_id"), "email": a.get("email")}
for a in new_manifest.get("accounts", [])
]
manifest_accounts_excluding = [
a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id
]
# handle all accounts
sct_manifest_accounts = json.dumps(manifest_accounts_all)
sct_manifest_spokes = json.dumps(manifest_accounts_excluding)
regions = config.get_regions(puppet_account_id)
sct_config_regions = json.dumps(regions)
new_manifest["parameters"]["SCTManifestAccounts"] = dict(
default=sct_manifest_accounts
)
new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes)
new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions)
new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id)
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=regions,
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml_utils.dump(new_manifest))
def get_task_defs_from_details(launch_details, accounts, include_expanded_from,
launch_name, configuration):
tasks = []
deploy_to = launch_details.get('deploy_to')
for tag_list_item in deploy_to.get('tags', []):
for account in accounts:
for tag in account.get('tags', []):
if tag == tag_list_item.get('tag'):
tag_account_def = deepcopy(configuration)
tag_account_def['account_id'] = account.get('account_id')
tag_account_def['expanded_from'] = account.get(
'expanded_from', '')
tag_account_def['organization'] = account.get(
'organization')
regions = tag_list_item.get('regions', 'default_region')
if isinstance(regions, str):
if regions in [
"enabled", "regions_enabled", "enabled_regions"
]:
for region_enabled in account.get(
'regions_enabled'):
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def[
'region'] = region_enabled
tasks.append(region_tag_account_def)
elif regions == 'default_region':
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = account.get(
'default_region')
tasks.append(region_tag_account_def)
elif regions == "all":
all_regions = config.get_regions()
for region_enabled in all_regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def[
'region'] = region_enabled
tasks.append(region_tag_account_def)
else:
raise Exception(
f"Unsupported regions {regions} setting for launch: {launch_name}"
)
elif isinstance(regions, list):
for region in regions:
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region
tasks.append(region_tag_account_def)
elif isinstance(regions, tuple):
for region in regions:
region_tag_account_def = deepcopy(tag_account_def)
region_tag_account_def['region'] = region
tasks.append(region_tag_account_def)
else:
raise Exception(
f"Unexpected regions of {regions} set for launch {launch_name}"
)
for account_list_item in deploy_to.get('accounts', []):
for account in accounts:
if account.get('account_id') == account_list_item.get(
'account_id'):
account_account_def = deepcopy(configuration)
account_account_def['account_id'] = account.get('account_id')
account_account_def['expanded_from'] = account.get(
'expanded_from')
account_account_def['organization'] = account.get(
'organization')
# account_account_def['account_parameters'] = account.get('parameters', {})
regions = account_list_item.get('regions', 'default_region')
if isinstance(regions, str):
if regions in [
"enabled", "regions_enabled", "enabled_regions"
]:
for region_enabled in account.get('regions_enabled'):
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def[
'region'] = region_enabled
tasks.append(region_account_account_def)
elif regions == 'default_region':
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def['region'] = account.get(
'default_region')
tasks.append(region_account_account_def)
elif regions == "all":
all_regions = config.get_regions()
for region_enabled in all_regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def[
'region'] = region_enabled
tasks.append(region_account_account_def)
else:
raise Exception(
f"Unsupported regions {regions} setting for launch: {launch_name}"
)
elif isinstance(regions, list):
for region in regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def['region'] = region
tasks.append(region_account_account_def)
elif isinstance(regions, tuple):
for region in regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def['region'] = region
tasks.append(region_account_account_def)
else:
raise Exception(
f"Unexpected regions of {regions} set for launch {launch_name}"
)
return tasks
def run_tasks_for_generate_shares(tasks_to_run):
for type in [
"failure",
"success",
"timeout",
"process_failure",
"processing_time",
"broken_task",
]:
os.makedirs(Path(constants.RESULTS_DIRECTORY) / type)
run_result = luigi.build(
tasks_to_run,
local_scheduler=True,
detailed_summary=True,
workers=10,
log_level='INFO',
)
should_use_sns = config.get_should_use_sns()
puppet_account_id = config.get_puppet_account_id()
version = config.get_puppet_version()
for region in config.get_regions():
sharing_policies = {
'accounts': [],
'organizations': [],
}
with betterboto_client.ClientContextManager(
'cloudformation', region_name=region) as cloudformation:
cloudformation.ensure_deleted(
StackName="servicecatalog-puppet-shares")
logger.info(f"generating policies collection for region {region}")
if os.path.exists(os.path.sep.join(['data', 'bucket'])):
logger.info(f"Updating policies for the region: {region}")
path = os.path.sep.join(['data', 'bucket', region, 'accounts'])
if os.path.exists(path):
for account_file in os.listdir(path):
account = account_file.split(".")[0]
sharing_policies['accounts'].append(account)
path = os.path.sep.join(
['data', 'bucket', region, 'organizations'])
if os.path.exists(path):
for organization_file in os.listdir(path):
organization = organization_file.split(".")[0]
sharing_policies['organizations'].append(organization)
logger.info(f"Finished generating policies collection")
template = config.env.get_template(
'policies.template.yaml.j2').render(
sharing_policies=sharing_policies,
VERSION=version,
)
with betterboto_client.ClientContextManager(
'cloudformation', region_name=region) as cloudformation:
cloudformation.create_or_update(
StackName="servicecatalog-puppet-policies",
TemplateBody=template,
NotificationARNs=[
f"arn:aws:sns:{region}:{puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events"
] if should_use_sns else [],
)
for filename in glob('results/failure/*.json'):
result = json.loads(open(filename, 'r').read())
click.echo(
colorclass.Color("{red}" + result.get('task_type') +
" failed{/red}"))
click.echo(
f"{yaml.safe_dump({'parameters':result.get('task_params')})}")
click.echo("\n".join(result.get('exception_stack_trace')))
click.echo('')
exit_status_codes = {
LuigiStatusCode.SUCCESS: 0,
LuigiStatusCode.SUCCESS_WITH_RETRY: 0,
LuigiStatusCode.FAILED: 1,
LuigiStatusCode.FAILED_AND_SCHEDULING_FAILED: 2,
LuigiStatusCode.SCHEDULING_FAILED: 3,
LuigiStatusCode.NOT_RUN: 4,
LuigiStatusCode.MISSING_EXT: 5,
}
sys.exit(exit_status_codes.get(run_result.status))
def get_task_defs_from_details(
self,
puppet_account_id,
launch_name,
configuration,
launch_or_spoke_local_portfolio,
):
launch_details = self.get(launch_or_spoke_local_portfolio).get(
launch_name)
accounts = self.get("accounts")
if launch_details is None:
raise Exception(f"launch_details is None for {launch_name}")
if launch_or_spoke_local_portfolio == "lambda-invocations":
deploy_to = launch_details.get("invoke_for")
elif launch_or_spoke_local_portfolio == "launches":
deploy_to = launch_details.get("deploy_to")
elif launch_or_spoke_local_portfolio == "spoke-local-portfolios":
deploy_to = launch_details.get("deploy_to") or launch_details.get(
"share_with")
task_defs = []
for tag_list_item in deploy_to.get("tags", []):
for account in accounts:
for tag in account.get("tags", []):
if tag == tag_list_item.get("tag"):
tag_account_def = deepcopy(configuration)
tag_account_def["account_id"] = account.get(
"account_id")
tag_account_def["account_parameters"] = account.get(
"parameters", {})
regions = tag_list_item.get("regions",
"default_region")
if isinstance(regions, str):
if regions in [
"enabled",
"regions_enabled",
"enabled_regions",
]:
for region_enabled in account.get(
"regions_enabled"):
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def[
"region"] = region_enabled
task_defs.append(region_tag_account_def)
elif regions == "default_region":
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def["region"] = account.get(
"default_region")
task_defs.append(region_tag_account_def)
elif regions == "all":
all_regions = config.get_regions(
puppet_account_id)
for region_enabled in all_regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def[
"region"] = region_enabled
task_defs.append(region_tag_account_def)
else:
raise Exception(
f"Unsupported regions {regions} setting for launch: {launch_name}"
)
elif isinstance(regions, list):
for region in regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def["region"] = region
task_defs.append(region_tag_account_def)
elif isinstance(regions, tuple):
for region in regions:
region_tag_account_def = deepcopy(
tag_account_def)
region_tag_account_def["region"] = region
task_defs.append(region_tag_account_def)
else:
raise Exception(
f"Unexpected regions of {regions} set for launch {launch_name}"
)
for account_list_item in deploy_to.get("accounts", []):
for account in accounts:
if account.get("account_id") == account_list_item.get(
"account_id"):
account_account_def = deepcopy(configuration)
account_account_def["account_id"] = account.get(
"account_id")
account_account_def["account_parameters"] = account.get(
"parameters", {})
regions = account_list_item.get("regions",
"default_region")
if isinstance(regions, str):
if regions in [
"enabled", "regions_enabled", "enabled_regions"
]:
for region_enabled in account.get(
"regions_enabled"):
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def[
"region"] = region_enabled
task_defs.append(region_account_account_def)
elif regions in ["default_region"]:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def["region"] = account.get(
"default_region")
task_defs.append(region_account_account_def)
elif regions in ["all"]:
all_regions = config.get_regions(puppet_account_id)
for region_enabled in all_regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def[
"region"] = region_enabled
task_defs.append(region_account_account_def)
else:
raise Exception(
f"Unsupported regions {regions} setting for launch: {launch_name}"
)
elif isinstance(regions, list):
for region in regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def["region"] = region
task_defs.append(region_account_account_def)
elif isinstance(regions, tuple):
for region in regions:
region_account_account_def = deepcopy(
account_account_def)
region_account_account_def["region"] = region
task_defs.append(region_account_account_def)
else:
raise Exception(
f"Unexpected regions of {regions} set for launch {launch_name}"
)
return task_defs
def get_tasks_for(self,
puppet_account_id,
section_name,
item_name,
single_account="None"):
accounts = self.get(constants.ACCOUNTS)
section = self.get(section_name)
provisioning_tasks = list()
item = section[item_name]
deploy_to = {
"launches": "deploy_to",
"stacks": "deploy_to",
"apps": "deploy_to",
"workspaces": "deploy_to",
"spoke-local-portfolios": "share_with",
"lambda-invocations": "invoke_for",
"code-build-runs": "run_for",
"assertions": "assert_for",
}.get(section_name)
if (section_name == constants.SPOKE_LOCAL_PORTFOLIOS
and item.get(deploy_to) is None):
deploy_to = "deploy_to"
common_parameters = {
"launches":
dict(
puppet_account_id=puppet_account_id,
launch_name=item_name,
launch_parameters=item.get("parameters", {}),
manifest_parameters=self.get("parameters", {}),
ssm_param_outputs=item.get("outputs", {}).get("ssm", []),
portfolio=item.get("portfolio"),
product=item.get("product"),
version=item.get("version"),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
requested_priority=item.get("requested_priority", 0),
),
"stacks":
dict(
puppet_account_id=puppet_account_id,
stack_name=item_name,
launch_name=item.get("launch_name", ""),
stack_set_name=item.get("stack_set_name", ""),
launch_parameters=item.get("parameters", {}),
capabilities=item.get("capabilities", []),
manifest_parameters=self.get("parameters", {}),
ssm_param_outputs=item.get("outputs", {}).get("ssm", []),
bucket=f"sc-puppet-stacks-repository-{puppet_account_id}",
key=item.get("key"),
version_id=item.get("version_id", ""),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
use_service_role=item.get(
constants.MANIFEST_SHOULD_USE_STACKS_SERVICE_ROLE,
constants.CONFIG_SHOULD_USE_STACKS_SERVICE_ROLE_DEFAULT,
),
requested_priority=item.get("requested_priority", 0),
),
"apps":
dict(
puppet_account_id=puppet_account_id,
app_name=item_name,
launch_parameters=item.get("parameters", {}),
manifest_parameters=self.get("parameters", {}),
ssm_param_outputs=item.get("outputs", {}).get("ssm", []),
bucket=f"sc-puppet-stacks-repository-{puppet_account_id}",
key=item.get("key"),
version_id=item.get("version_id", ""),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
requested_priority=item.get("requested_priority", 0),
),
"workspaces":
dict(
puppet_account_id=puppet_account_id,
workspace_name=item_name,
launch_parameters=item.get("parameters", {}),
manifest_parameters=self.get("parameters", {}),
ssm_param_outputs=item.get("outputs", {}).get("ssm", []),
bucket=f"sc-puppet-stacks-repository-{puppet_account_id}",
key=item.get("key"),
version_id=item.get("version_id", ""),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
requested_priority=item.get("requested_priority", 0),
),
"spoke-local-portfolios":
dict(
puppet_account_id=puppet_account_id,
spoke_local_portfolio_name=item_name,
product_generation_method=item.get(
"product_generation_method",
constants.PRODUCT_GENERATION_METHOD_DEFAULT,
),
organization=item.get("organization", ""),
sharing_mode=item.get("sharing_mode",
constants.SHARING_MODE_DEFAULT),
associations=item.get("associations", list()),
launch_constraints=item.get("constraints",
{}).get("launch", []),
portfolio=item.get("portfolio"),
),
"lambda-invocations":
dict(
puppet_account_id=puppet_account_id,
lambda_invocation_name=item_name,
function_name=item.get("function_name"),
qualifier=item.get("qualifier", "$LATEST"),
invocation_type=item.get("invocation_type"),
launch_parameters=item.get("parameters", {}),
manifest_parameters=self.get("parameters", {}),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
),
"code-build-runs":
dict(
puppet_account_id=puppet_account_id,
code_build_run_name=item_name,
launch_parameters=item.get("parameters", {}),
manifest_parameters=self.get("parameters", {}),
project_name=item.get("project_name"),
requested_priority=item.get("requested_priority", 0),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
),
"assertions":
dict(
puppet_account_id=puppet_account_id,
requested_priority=item.get("requested_priority", 0),
assertion_name=item_name,
expected=item.get("expected"),
actual=item.get("actual"),
execution=item.get("execution",
constants.EXECUTION_MODE_DEFAULT),
),
}.get(section_name)
tags = item.get(deploy_to).get("tags", [])
for tag in tags:
tag_name = tag.get("tag")
regions = tag.get("regions")
for account in accounts:
account_id = str(account.get("account_id"))
if single_account != "None" and single_account != account_id:
continue
additional_parameters = {
"launches":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"apps":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"workspaces":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"stacks":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"spoke-local-portfolios":
dict(account_id=account_id, ),
"assertions":
dict(account_id=account_id, ),
"lambda-invocations":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"code-build-runs":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
}.get(section_name)
if tag_name in account.get("tags"):
if isinstance(regions, str):
if regions in [
"enabled",
"regions_enabled",
"enabled_regions",
]:
for region_enabled in account.get(
"regions_enabled"):
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_enabled,
))
elif regions == "default_region":
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=account.get("default_region"),
))
elif regions == "all":
all_regions = config.get_regions(puppet_account_id)
for region_enabled in all_regions:
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_enabled,
))
else:
raise Exception(
f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}"
)
elif isinstance(regions, list):
for region_ in regions:
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_,
))
elif isinstance(regions, tuple):
for region_ in regions:
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_,
))
else:
raise Exception(
f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}"
)
for account_to_deploy_to in item.get(deploy_to).get("accounts", []):
account_id_of_account_to_deploy_to = account_to_deploy_to.get(
"account_id")
regions = account_to_deploy_to.get("regions")
account = self.get_account(account_id_of_account_to_deploy_to)
account_id = account_id_of_account_to_deploy_to
if single_account != "None" and single_account != account_id:
continue
additional_parameters = {
"launches":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"stacks":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"spoke-local-portfolios":
dict(account_id=account_id, ),
"assertions":
dict(account_id=account_id, ),
"lambda-invocations":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
"code-build-runs":
dict(
account_id=account_id,
account_parameters=account.get("parameters", {}),
),
}.get(section_name)
if isinstance(regions, str):
if regions in [
"enabled",
"regions_enabled",
"enabled_regions",
]:
for region_enabled in account.get("regions_enabled"):
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_enabled,
))
elif regions == "default_region":
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=account.get("default_region"),
))
elif regions == "all":
all_regions = config.get_regions(puppet_account_id)
for region_enabled in all_regions:
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_enabled,
))
else:
raise Exception(
f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}"
)
elif isinstance(regions, list):
for region_ in regions:
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_,
))
elif isinstance(regions, tuple):
for region_ in regions:
provisioning_tasks.append(
dict(
**common_parameters,
**additional_parameters,
region=region_,
))
else:
raise Exception(
f"Unsupported regions {regions} setting for {constants.LAUNCHES}: {item_name}"
)
return provisioning_tasks
