def output_location(self):
puppet_account_id = config.get_puppet_account_id()
path = f"output/{self.uid}.{self.output_suffix}"
if config.is_caching_enabled(puppet_account_id):
return f"s3://sc-puppet-caching-bucket-{config.get_puppet_account_id()}-{config.get_home_region(puppet_account_id)}/{path}"
else:
return path
def output(self):
should_use_s3_target_if_caching_is_on = (
"cache_invalidator"
not in self.params_for_results_display().keys())
if should_use_s3_target_if_caching_is_on and config.is_caching_enabled(
config.get_puppet_account_id()):
return s3.S3Target(self.output_location, format=format.UTF8)
else:
return luigi.LocalTarget(self.output_location)
def output_location(self):
puppet_account_id = config.get_puppet_account_id()
path = f"output/{self.uid}.{self.output_suffix}"
should_use_s3_target_if_caching_is_on = (
"cache_invalidator"
not in self.params_for_results_display().keys())
if should_use_s3_target_if_caching_is_on and config.is_caching_enabled(
puppet_account_id):
return f"s3://sc-puppet-caching-bucket-{config.get_puppet_account_id()}-{config.get_home_region(puppet_account_id)}/{path}"
else:
return path
def bootstrap(
puppet_account_id,
with_manual_approvals,
puppet_code_pipeline_role_permission_boundary,
source_role_permissions_boundary,
puppet_generate_role_permission_boundary,
puppet_deploy_role_permission_boundary,
puppet_provisioning_role_permissions_boundary,
cloud_formation_deploy_role_permissions_boundary,
deploy_environment_compute_type,
spoke_deploy_environment_compute_type,
deploy_num_workers,
source_provider,
owner,
repo,
branch,
poll_for_source_changes,
webhook_secret,
puppet_role_name,
puppet_role_path,
scm_connection_arn,
scm_full_repository_id,
scm_branch_name,
scm_bucket_name,
scm_object_key,
scm_skip_creation_of_repo,
should_validate,
custom_source_action_git_url,
custom_source_action_git_web_hook_ip_address,
custom_source_action_custom_action_type_version,
custom_source_action_custom_action_type_provider,
):
click.echo("Starting bootstrap")
should_use_eventbridge = config.get_should_use_eventbridge(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
initialiser_stack_tags = config.get_initialiser_stack_tags()
if should_use_eventbridge:
with betterboto_client.ClientContextManager("events") as events:
try:
events.describe_event_bus(Name=constants.EVENT_BUS_NAME)
except events.exceptions.ResourceNotFoundException:
events.create_event_bus(Name=constants.EVENT_BUS_NAME,)
all_regions = config.get_regions(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
with betterboto_client.MultiRegionClientContextManager(
"cloudformation", all_regions
) as clients:
click.echo("Creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME))
threads = []
template = hub_bootstrap_region.get_template(
constants.VERSION, os.environ.get("AWS_DEFAULT_REGION")
).to_yaml(clean_up=True)
args = {
"StackName": "{}-regional".format(constants.BOOTSTRAP_STACK_NAME),
"TemplateBody": template,
"Capabilities": ["CAPABILITY_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": constants.VERSION,
"UsePreviousValue": False,
},
{
"ParameterKey": "DefaultRegionValue",
"ParameterValue": os.environ.get("AWS_DEFAULT_REGION"),
"UsePreviousValue": False,
},
],
"Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}]
+ initialiser_stack_tags,
}
for client_region, client in clients.items():
process = Thread(
name=client_region, target=client.create_or_update, kwargs=args
)
process.start()
threads.append(process)
for process in threads:
process.join()
click.echo(
"Finished creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME)
)
source_args = {"Provider": source_provider}
if source_provider == "CodeCommit":
source_args.update(
{
"Configuration": {
"RepositoryName": repo,
"BranchName": branch,
"PollForSourceChanges": poll_for_source_changes,
},
}
)
elif source_provider == "GitHub":
source_args.update(
{
"Configuration": {
"Owner": owner,
"Repo": repo,
"Branch": branch,
"PollForSourceChanges": poll_for_source_changes,
"SecretsManagerSecret": webhook_secret,
},
}
)
elif source_provider.lower() == "codestarsourceconnection":
source_args.update(
{
"Configuration": {
"ConnectionArn": scm_connection_arn,
"FullRepositoryId": scm_full_repository_id,
"BranchName": scm_branch_name,
"OutputArtifactFormat": "CODE_ZIP",
"DetectChanges": poll_for_source_changes,
},
}
)
elif source_provider.lower() == "s3":
source_args.update(
{
"Configuration": {
"S3Bucket": scm_bucket_name,
"S3ObjectKey": scm_object_key,
"PollForSourceChanges": poll_for_source_changes,
},
}
)
elif source_provider.lower() == "custom":
source_args.update(
{
"Configuration": {
"Owner": "Custom",
"GitUrl": custom_source_action_git_url,
"Branch": branch,
"GitWebHookIpAddress": custom_source_action_git_web_hook_ip_address,
"CustomActionTypeVersion": custom_source_action_custom_action_type_version,
"CustomActionTypeProvider": custom_source_action_custom_action_type_provider,
},
}
)
template = hub_bootstrap.get_template(
constants.VERSION,
all_regions,
source_args,
config.is_caching_enabled(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
),
with_manual_approvals,
scm_skip_creation_of_repo,
should_validate,
).to_yaml(clean_up=True)
args = {
"StackName": constants.BOOTSTRAP_STACK_NAME,
"TemplateBody": template,
"Capabilities": ["CAPABILITY_NAMED_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": constants.VERSION,
"UsePreviousValue": False,
},
{
"ParameterKey": "OrgIamRoleArn",
"ParameterValue": str(config.get_org_iam_role_arn(puppet_account_id)),
"UsePreviousValue": False,
},
{
"ParameterKey": "WithManualApprovals",
"ParameterValue": "Yes" if with_manual_approvals else "No",
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetCodePipelineRolePermissionBoundary",
"ParameterValue": puppet_code_pipeline_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "SourceRolePermissionsBoundary",
"ParameterValue": source_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetGenerateRolePermissionBoundary",
"ParameterValue": puppet_generate_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetDeployRolePermissionBoundary",
"ParameterValue": puppet_deploy_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetProvisioningRolePermissionsBoundary",
"ParameterValue": puppet_provisioning_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "CloudFormationDeployRolePermissionsBoundary",
"ParameterValue": cloud_formation_deploy_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployEnvironmentComputeType",
"ParameterValue": deploy_environment_compute_type,
"UsePreviousValue": False,
},
{
"ParameterKey": "SpokeDeployEnvironmentComputeType",
"ParameterValue": spoke_deploy_environment_compute_type,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployNumWorkers",
"ParameterValue": str(deploy_num_workers),
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetRoleName",
"ParameterValue": puppet_role_name,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetRolePath",
"ParameterValue": puppet_role_path,
"UsePreviousValue": False,
},
],
"Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}]
+ initialiser_stack_tags,
}
with betterboto_client.ClientContextManager("cloudformation") as cloudformation:
click.echo("Creating {}".format(constants.BOOTSTRAP_STACK_NAME))
cloudformation.create_or_update(**args)
buff = io.BytesIO()
with zipfile.ZipFile(buff, mode="w", compression=zipfile.ZIP_DEFLATED) as z:
z.writestr("parameters.yaml", 'single_account: "000000000000"')
with betterboto_client.ClientContextManager("s3") as s3:
try:
s3.head_object(
Bucket=f"sc-puppet-parameterised-runs-{puppet_account_id}",
Key="parameters.zip",
)
except botocore.exceptions.ClientError as ex:
if ex.response["Error"]["Code"] == "404":
s3.put_object(
Bucket=f"sc-puppet-parameterised-runs-{puppet_account_id}",
Key="parameters.zip",
Body=buff.getvalue(),
)
click.echo("Finished creating {}.".format(constants.BOOTSTRAP_STACK_NAME))
def _do_bootstrap(
puppet_version,
puppet_account_id,
with_manual_approvals,
puppet_code_pipeline_role_permission_boundary,
source_role_permissions_boundary,
puppet_generate_role_permission_boundary,
puppet_deploy_role_permission_boundary,
puppet_provisioning_role_permissions_boundary,
cloud_formation_deploy_role_permissions_boundary,
deploy_environment_compute_type,
deploy_num_workers,
source_provider,
owner,
repo,
branch,
poll_for_source_changes,
webhook_secret,
puppet_role_name,
puppet_role_path,
):
click.echo("Starting bootstrap")
should_use_eventbridge = config.get_should_use_eventbridge(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
if should_use_eventbridge:
with betterboto_client.ClientContextManager("events") as events:
try:
events.describe_event_bus(Name=constants.EVENT_BUS_NAME)
except events.exceptions.ResourceNotFoundException:
events.create_event_bus(Name=constants.EVENT_BUS_NAME,)
all_regions = config.get_regions(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
with betterboto_client.MultiRegionClientContextManager(
"cloudformation", all_regions
) as clients:
click.echo("Creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME))
threads = []
template = asset_helpers.read_from_site_packages(
"{}.template.yaml".format(
"{}-regional".format(constants.BOOTSTRAP_STACK_NAME)
)
)
template = Template(template).render(VERSION=puppet_version)
args = {
"StackName": "{}-regional".format(constants.BOOTSTRAP_STACK_NAME),
"TemplateBody": template,
"Capabilities": ["CAPABILITY_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": puppet_version,
"UsePreviousValue": False,
},
{
"ParameterKey": "DefaultRegionValue",
"ParameterValue": os.environ.get("AWS_DEFAULT_REGION"),
"UsePreviousValue": False,
},
],
"Tags": [{"Key": "ServiceCatalogPuppet:Actor", "Value": "Framework",}],
}
for client_region, client in clients.items():
process = Thread(
name=client_region, target=client.create_or_update, kwargs=args
)
process.start()
threads.append(process)
for process in threads:
process.join()
click.echo(
"Finished creating {}-regional".format(constants.BOOTSTRAP_STACK_NAME)
)
source_args = {"Provider": source_provider}
if source_provider == "CodeCommit":
source_args.update(
{"Configuration": {"RepositoryName": repo, "BranchName": branch,},}
)
elif source_provider == "GitHub":
source_args.update(
{
"Configuration": {
"Owner": owner,
"Repo": repo,
"Branch": branch,
"PollForSourceChanges": poll_for_source_changes,
"SecretsManagerSecret": webhook_secret,
},
}
)
with betterboto_client.ClientContextManager("cloudformation") as cloudformation:
click.echo("Creating {}".format(constants.BOOTSTRAP_STACK_NAME))
template = asset_helpers.read_from_site_packages(
"{}.template.yaml".format(constants.BOOTSTRAP_STACK_NAME)
)
template = Template(template).render(
VERSION=puppet_version,
ALL_REGIONS=all_regions,
Source=source_args,
is_caching_enabled=config.is_caching_enabled(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
),
)
template = Template(template).render(
VERSION=puppet_version, ALL_REGIONS=all_regions, Source=source_args
)
args = {
"StackName": constants.BOOTSTRAP_STACK_NAME,
"TemplateBody": template,
"Capabilities": ["CAPABILITY_NAMED_IAM"],
"Parameters": [
{
"ParameterKey": "Version",
"ParameterValue": puppet_version,
"UsePreviousValue": False,
},
{
"ParameterKey": "OrgIamRoleArn",
"ParameterValue": str(
config.get_org_iam_role_arn(puppet_account_id)
),
"UsePreviousValue": False,
},
{
"ParameterKey": "WithManualApprovals",
"ParameterValue": "Yes" if with_manual_approvals else "No",
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetCodePipelineRolePermissionBoundary",
"ParameterValue": puppet_code_pipeline_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "SourceRolePermissionsBoundary",
"ParameterValue": source_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetGenerateRolePermissionBoundary",
"ParameterValue": puppet_generate_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetDeployRolePermissionBoundary",
"ParameterValue": puppet_deploy_role_permission_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetProvisioningRolePermissionsBoundary",
"ParameterValue": puppet_provisioning_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "CloudFormationDeployRolePermissionsBoundary",
"ParameterValue": cloud_formation_deploy_role_permissions_boundary,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployEnvironmentComputeType",
"ParameterValue": deploy_environment_compute_type,
"UsePreviousValue": False,
},
{
"ParameterKey": "DeployNumWorkers",
"ParameterValue": str(deploy_num_workers),
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetRoleName",
"ParameterValue": puppet_role_name,
"UsePreviousValue": False,
},
{
"ParameterKey": "PuppetRolePath",
"ParameterValue": puppet_role_path,
"UsePreviousValue": False,
},
],
}
cloudformation.create_or_update(**args)
click.echo("Finished creating {}.".format(constants.BOOTSTRAP_STACK_NAME))
if source_provider == "CodeCommit":
with betterboto_client.ClientContextManager("codecommit") as codecommit:
response = codecommit.get_repository(repositoryName=repo)
clone_url = response.get("repositoryMetadata").get("cloneUrlHttp")
clone_command = (
"git clone --config 'credential.helper=!aws codecommit credential-helper $@' "
"--config 'credential.UseHttpPath=true' {}".format(clone_url)
)
click.echo(
"You need to clone your newly created repo now and will then need to seed it: \n{}".format(
clone_command
)
)
def output(self):
if config.is_caching_enabled(config.get_puppet_account_id()):
return s3.S3Target(self.output_location, format=format.UTF8)
else:
return luigi.LocalTarget(self.output_location)
