def initialize(template, service_name, environment='dev'):
"""Adds SERVICE_NAME, SERVICE_ENVIRONMENT, and DEFAULT_TAGS to the template
:param template:
:param service_name:
:param environment:
:return:
"""
template.SERVICE_NAME = os.getenv('SERVICE_NAME', service_name)
template.SERVICE_ENVIRONMENT = os.getenv('ENV', environment).lower()
template.DEFAULT_TAGS = troposphere.Tags(**{
'service-name': template.SERVICE_NAME,
'environment': template.SERVICE_ENVIRONMENT
})
template.add_version("2010-09-09")
template.add_description("Stack for %s microservice" % service_name)
def _create_lambda_function(self, code_property, task_function, role,
runtime):
# type: (awslambda.Code, TaskFunction, iam.Role, str) -> None
# TODO add support for versioning
function_handler = self._get_function_handler_string(
task_function.func)
title = self._get_function_logical_id(function_handler)
function_kwargs = {
"Code":
code_property,
"Handler":
function_handler,
"Role":
GetAtt(role, "Arn"),
"Runtime":
runtime,
"Environment":
awslambda.Environment(
Variables=task_function.environment_variables),
"Tags":
troposphere.Tags(task_function.tags),
}
if self._app.kms_key_arn is not None and len(
self._app.kms_key_arn) > 0:
function_kwargs["KmsKeyArn"] = self._app.kms_key_arn
if len(self._app.subnet_ids) > 0 or len(
self._app.security_group_ids) > 0:
function_kwargs["VpcConfig"] = awslambda.VPCConfig(
SubnetIds=self._app.subnet_ids,
SecurityGroupIds=self._app.security_group_ids)
if task_function.memory is not None:
function_kwargs["MemorySize"] = task_function.memory
if task_function.timeout is not None:
function_kwargs["Timeout"] = task_function.timeout
if task_function.activate_tracing:
function_kwargs["TracingConfig"] = awslambda.TracingConfig(
Mode="Active")
# TODO specify the function name? Maybe we don't care?
return awslambda.Function(title, **function_kwargs)
def buildCloudFormationTemplate():
global t
global parameters
global applicationTags
t = Template()
parameters = {}
t.add_version("2010-09-09")
t.add_description("Deploys necessary s3 buckets for EDW POC")
applicationTags = troposphere.Tags(
Application="Enterprise Data Warehouse POC",
Environment="dev",
Team="EDMS")
try:
addParameters()
addConditions()
addResources()
createFiles(TEMPLATE_NAME)
except Exception as e:
print e
def get_template(
puppet_version,
all_regions,
source,
is_caching_enabled,
is_manual_approvals: bool,
scm_skip_creation_of_repo: bool,
should_validate: bool,
) -> t.Template:
is_codecommit = source.get("Provider", "").lower() == "codecommit"
is_github = source.get("Provider", "").lower() == "github"
is_codestarsourceconnection = (source.get(
"Provider", "").lower() == "codestarsourceconnection")
is_custom = (source.get("Provider", "").lower() == "custom")
is_s3 = source.get("Provider", "").lower() == "s3"
description = f"""Bootstrap template used to bring up the main ServiceCatalog-Puppet AWS CodePipeline with dependencies
{{"version": "{puppet_version}", "framework": "servicecatalog-puppet", "role": "bootstrap-master"}}"""
template = t.Template(Description=description)
version_parameter = template.add_parameter(
t.Parameter("Version", Default=puppet_version, Type="String"))
org_iam_role_arn_parameter = template.add_parameter(
t.Parameter("OrgIamRoleArn", Default="None", Type="String"))
with_manual_approvals_parameter = template.add_parameter(
t.Parameter(
"WithManualApprovals",
Type="String",
AllowedValues=["Yes", "No"],
Default="No",
))
puppet_code_pipeline_role_permission_boundary_parameter = template.add_parameter(
t.Parameter(
"PuppetCodePipelineRolePermissionBoundary",
Type="String",
Description=
"IAM Permission Boundary to apply to the PuppetCodePipelineRole",
Default=awscs_iam.ARN(resource="policy/AdministratorAccess").data,
))
source_role_permissions_boundary_parameter = template.add_parameter(
t.Parameter(
"SourceRolePermissionsBoundary",
Type="String",
Description="IAM Permission Boundary to apply to the SourceRole",
Default=awscs_iam.ARN(resource="policy/AdministratorAccess").data,
))
puppet_generate_role_permission_boundary_parameter = template.add_parameter(
t.Parameter(
"PuppetGenerateRolePermissionBoundary",
Type="String",
Description=
"IAM Permission Boundary to apply to the PuppetGenerateRole",
Default=awscs_iam.ARN(resource="policy/AdministratorAccess").data,
))
puppet_deploy_role_permission_boundary_parameter = template.add_parameter(
t.Parameter(
"PuppetDeployRolePermissionBoundary",
Type="String",
Description=
"IAM Permission Boundary to apply to the PuppetDeployRole",
Default=awscs_iam.ARN(resource="policy/AdministratorAccess").data,
))
puppet_provisioning_role_permissions_boundary_parameter = template.add_parameter(
t.Parameter(
"PuppetProvisioningRolePermissionsBoundary",
Type="String",
Description=
"IAM Permission Boundary to apply to the PuppetProvisioningRole",
Default=awscs_iam.ARN(resource="policy/AdministratorAccess").data,
))
cloud_formation_deploy_role_permissions_boundary_parameter = template.add_parameter(
t.Parameter(
"CloudFormationDeployRolePermissionsBoundary",
Type="String",
Description=
"IAM Permission Boundary to apply to the CloudFormationDeployRole",
Default=awscs_iam.ARN(resource="policy/AdministratorAccess").data,
))
deploy_environment_compute_type_parameter = template.add_parameter(
t.Parameter(
"DeployEnvironmentComputeType",
Type="String",
Description="The AWS CodeBuild Environment Compute Type",
Default="BUILD_GENERAL1_SMALL",
))
spoke_deploy_environment_compute_type_parameter = template.add_parameter(
t.Parameter(
"SpokeDeployEnvironmentComputeType",
Type="String",
Description=
"The AWS CodeBuild Environment Compute Type for spoke execution mode",
Default="BUILD_GENERAL1_SMALL",
))
deploy_num_workers_parameter = template.add_parameter(
t.Parameter(
"DeployNumWorkers",
Type="Number",
Description=
"Number of workers that should be used when running a deploy",
Default=10,
))
puppet_role_name_parameter = template.add_parameter(
t.Parameter("PuppetRoleName", Type="String", Default="PuppetRole"))
puppet_role_path_template_parameter = template.add_parameter(
t.Parameter("PuppetRolePath",
Type="String",
Default="/servicecatalog-puppet/"))
template.add_condition(
"ShouldUseOrgs",
t.Not(t.Equals(t.Ref(org_iam_role_arn_parameter), "None")))
template.add_condition(
"HasManualApprovals",
t.Equals(t.Ref(with_manual_approvals_parameter), "Yes"))
template.add_resource(
s3.Bucket(
"StacksRepository",
BucketName=t.Sub("sc-puppet-stacks-repository-${AWS::AccountId}"),
VersioningConfiguration=s3.VersioningConfiguration(
Status="Enabled"),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.
ServerSideEncryptionByDefault(SSEAlgorithm="AES256"))
]),
PublicAccessBlockConfiguration=s3.PublicAccessBlockConfiguration(
BlockPublicAcls=True,
BlockPublicPolicy=True,
IgnorePublicAcls=True,
RestrictPublicBuckets=True,
),
Tags=t.Tags({"ServiceCatalogPuppet:Actor": "Framework"}),
))
manual_approvals_param = template.add_resource(
ssm.Parameter(
"ManualApprovalsParam",
Type="String",
Name="/servicecatalog-puppet/manual-approvals",
Value=t.Ref(with_manual_approvals_parameter),
))
template.add_resource(
ssm.Parameter(
"SpokeDeployEnvParameter",
Type="String",
Name=constants.SPOKE_EXECUTION_MODE_DEPLOY_ENV_PARAMETER_NAME,
Value=t.Ref(spoke_deploy_environment_compute_type_parameter),
))
param = template.add_resource(
ssm.Parameter(
"Param",
Type="String",
Name="service-catalog-puppet-version",
Value=t.Ref(version_parameter),
))
partition_parameter = template.add_resource(
ssm.Parameter(
"PartitionParameter",
Type="String",
Name="/servicecatalog-puppet/partition",
Value=t.Ref("AWS::Partition"),
))
puppet_role_name_parameter = template.add_resource(
ssm.Parameter(
"PuppetRoleNameParameter",
Type="String",
Name="/servicecatalog-puppet/puppet-role/name",
Value=t.Ref(puppet_role_name_parameter),
))
puppet_role_path_parameter = template.add_resource(
ssm.Parameter(
"PuppetRolePathParameter",
Type="String",
Name="/servicecatalog-puppet/puppet-role/path",
Value=t.Ref(puppet_role_path_template_parameter),
))
share_accept_function_role = template.add_resource(
iam.Role(
"ShareAcceptFunctionRole",
RoleName="ShareAcceptFunctionRole",
ManagedPolicyArns=[
t.Sub(
"arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
)
],
Path=t.Ref(puppet_role_path_template_parameter),
Policies=[
iam.Policy(
PolicyName="ServiceCatalogActions",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Resource": {
"Fn::Sub":
"arn:${AWS::Partition}:iam::*:role${PuppetRolePath}${PuppetRoleName}"
},
"Effect": "Allow",
}],
},
)
],
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
},
}],
},
))
provisioning_role = template.add_resource(
iam.Role(
"ProvisioningRole",
RoleName="PuppetProvisioningRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["codebuild.amazonaws.com"]
},
},
{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"AWS": {
"Fn::Sub": "${AWS::AccountId}"
}
},
},
],
},
ManagedPolicyArns=[
t.Sub(
"arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess"
)
],
PermissionsBoundary=t.Ref(
puppet_provisioning_role_permissions_boundary_parameter),
Path=t.Ref(puppet_role_path_template_parameter),
))
cloud_formation_deploy_role = template.add_resource(
iam.Role(
"CloudFormationDeployRole",
RoleName="CloudFormationDeployRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["cloudformation.amazonaws.com"]
},
},
{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"AWS": {
"Fn::Sub": "${AWS::AccountId}"
}
},
},
],
},
ManagedPolicyArns=[
t.Sub(
"arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess"
)
],
PermissionsBoundary=t.Ref(
cloud_formation_deploy_role_permissions_boundary_parameter),
Path=t.Ref(puppet_role_path_template_parameter),
))
pipeline_role = template.add_resource(
iam.Role(
"PipelineRole",
RoleName="PuppetCodePipelineRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["codepipeline.amazonaws.com"]
},
}],
},
ManagedPolicyArns=[
t.Sub(
"arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess"
)
],
PermissionsBoundary=t.Ref(
puppet_code_pipeline_role_permission_boundary_parameter),
Path=t.Ref(puppet_role_path_template_parameter),
))
source_role = template.add_resource(
iam.Role(
"SourceRole",
RoleName="PuppetSourceRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["codepipeline.amazonaws.com"]
},
},
{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"AWS": {
"Fn::Sub":
"arn:${AWS::Partition}:iam::${AWS::AccountId}:root"
}
},
},
],
},
ManagedPolicyArns=[
t.Sub(
"arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess"
)
],
PermissionsBoundary=t.Ref(
source_role_permissions_boundary_parameter),
Path=t.Ref(puppet_role_path_template_parameter),
))
dry_run_notification_topic = template.add_resource(
sns.Topic(
"DryRunNotificationTopic",
DisplayName="service-catalog-puppet-dry-run-approvals",
TopicName="service-catalog-puppet-dry-run-approvals",
Condition="HasManualApprovals",
))
deploy_role = template.add_resource(
iam.Role(
"DeployRole",
RoleName="PuppetDeployRole",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["codebuild.amazonaws.com"]
},
}],
},
ManagedPolicyArns=[
t.Sub(
"arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess"
)
],
PermissionsBoundary=t.Ref(
puppet_deploy_role_permission_boundary_parameter),
Path=t.Ref(puppet_role_path_template_parameter),
))
num_workers_ssm_parameter = template.add_resource(
ssm.Parameter(
"NumWorkersSSMParameter",
Type="String",
Name="/servicecatalog-puppet/deploy/num-workers",
Value=t.Sub("${DeployNumWorkers}"),
))
parameterised_source_bucket = template.add_resource(
s3.Bucket(
"ParameterisedSourceBucket",
PublicAccessBlockConfiguration=s3.PublicAccessBlockConfiguration(
IgnorePublicAcls=True,
BlockPublicPolicy=True,
BlockPublicAcls=True,
RestrictPublicBuckets=True,
),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.
ServerSideEncryptionByDefault(SSEAlgorithm="AES256"))
]),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
BucketName=t.Sub("sc-puppet-parameterised-runs-${AWS::AccountId}"),
VersioningConfiguration=s3.VersioningConfiguration(
Status="Enabled"),
))
source_stage = codepipeline.Stages(
Name="Source",
Actions=[
codepipeline.Actions(
RunOrder=1,
RoleArn=t.GetAtt("SourceRole", "Arn"),
ActionTypeId=codepipeline.ActionTypeId(
Category="Source",
Owner="AWS",
Version="1",
Provider="S3",
),
OutputArtifacts=[
codepipeline.OutputArtifacts(Name="ParameterisedSource")
],
Configuration={
"S3Bucket": t.Ref(parameterised_source_bucket),
"S3ObjectKey": "parameters.zip",
"PollForSourceChanges": True,
},
Name="ParameterisedSource",
)
],
)
install_spec = {
"runtime-versions":
dict(python="3.7"),
"commands": [
f"pip install {puppet_version}" if "http" in puppet_version else
f"pip install aws-service-catalog-puppet=={puppet_version}",
],
}
deploy_env_vars = [
{
"Type": "PLAINTEXT",
"Name": "PUPPET_ACCOUNT_ID",
"Value": t.Ref("AWS::AccountId"),
},
{
"Type": "PLAINTEXT",
"Name": "PUPPET_REGION",
"Value": t.Ref("AWS::Region"),
},
{
"Type": "PARAMETER_STORE",
"Name": "PARTITION",
"Value": t.Ref(partition_parameter),
},
{
"Type": "PARAMETER_STORE",
"Name": "PUPPET_ROLE_NAME",
"Value": t.Ref(puppet_role_name_parameter),
},
{
"Type": "PARAMETER_STORE",
"Name": "PUPPET_ROLE_PATH",
"Value": t.Ref(puppet_role_path_parameter),
},
]
if is_codecommit:
template.add_resource(
codecommit.Repository(
"CodeRepo",
RepositoryName=source.get("Configuration").get(
"RepositoryName"),
RepositoryDescription=
"Repo to store the servicecatalog puppet solution",
DeletionPolicy="Retain",
))
source_stage.Actions.append(
codepipeline.Actions(
RunOrder=1,
RoleArn=t.GetAtt("SourceRole", "Arn"),
ActionTypeId=codepipeline.ActionTypeId(
Category="Source",
Owner="AWS",
Version="1",
Provider="CodeCommit",
),
OutputArtifacts=[codepipeline.OutputArtifacts(Name="Source")],
Configuration={
"RepositoryName":
source.get("Configuration").get("RepositoryName"),
"BranchName":
source.get("Configuration").get("BranchName"),
"PollForSourceChanges":
source.get("Configuration").get("PollForSourceChanges",
True),
},
Name="Source",
))
if is_github:
source_stage.Actions.append(
codepipeline.Actions(
RunOrder=1,
ActionTypeId=codepipeline.ActionTypeId(
Category="Source",
Owner="ThirdParty",
Version="1",
Provider="GitHub",
),
OutputArtifacts=[codepipeline.OutputArtifacts(Name="Source")],
Configuration={
"Owner":
source.get("Configuration").get("Owner"),
"Repo":
source.get("Configuration").get("Repo"),
"Branch":
source.get("Configuration").get("Branch"),
"OAuthToken":
t.Join(
"",
[
"{{resolve:secretsmanager:",
source.get("Configuration").get(
"SecretsManagerSecret"),
":SecretString:OAuthToken}}",
],
),
"PollForSourceChanges":
source.get("Configuration").get("PollForSourceChanges"),
},
Name="Source",
))
if is_custom:
source_stage.Actions.append(
codepipeline.Actions(
RunOrder=1,
ActionTypeId=codepipeline.ActionTypeId(
Category="Source",
Owner="Custom",
Version=source.get("Configuration").get(
"CustomActionTypeVersion"),
Provider=source.get("Configuration").get(
"CustomActionTypeProvider"),
),
OutputArtifacts=[codepipeline.OutputArtifacts(Name="Source")],
Configuration={
"GitUrl": source.get("Configuration").get("GitUrl"),
"Branch": source.get("Configuration").get("Branch"),
"PipelineName": t.Sub("${AWS::StackName}-pipeline"),
},
Name="Source",
))
webhook = codepipeline.Webhook(
"Webhook",
Authentication="IP",
TargetAction="Source",
AuthenticationConfiguration=codepipeline.WebhookAuthConfiguration(
AllowedIPRange=source.get("Configuration").get(
"GitWebHookIpAddress")),
Filters=[
codepipeline.WebhookFilterRule(
JsonPath="$.changes[0].ref.id",
MatchEquals="refs/heads/{Branch}")
],
TargetPipelineVersion=1,
TargetPipeline=t.Sub("${AWS::StackName}-pipeline"),
)
template.add_resource(webhook)
values_for_sub = {
"GitUrl": source.get("Configuration").get("GitUrl"),
"WebhookUrl": t.GetAtt(webhook, "Url"),
}
output_to_add = t.Output("WebhookUrl")
output_to_add.Value = t.Sub("${GitUrl}||${WebhookUrl}",
**values_for_sub)
output_to_add.Export = t.Export(t.Sub("${AWS::StackName}-pipeline"))
template.add_output(output_to_add)
if is_codestarsourceconnection:
source_stage.Actions.append(
codepipeline.Actions(
RunOrder=1,
RoleArn=t.GetAtt("SourceRole", "Arn"),
ActionTypeId=codepipeline.ActionTypeId(
Category="Source",
Owner="AWS",
Version="1",
Provider="CodeStarSourceConnection",
),
OutputArtifacts=[codepipeline.OutputArtifacts(Name="Source")],
Configuration={
"ConnectionArn":
source.get("Configuration").get("ConnectionArn"),
"FullRepositoryId":
source.get("Configuration").get("FullRepositoryId"),
"BranchName":
source.get("Configuration").get("BranchName"),
"OutputArtifactFormat":
source.get("Configuration").get("OutputArtifactFormat"),
},
Name="Source",
))
if is_s3:
bucket_name = source.get("Configuration").get("S3Bucket")
if not scm_skip_creation_of_repo:
template.add_resource(
s3.Bucket(
bucket_name,
PublicAccessBlockConfiguration=s3.
PublicAccessBlockConfiguration(
IgnorePublicAcls=True,
BlockPublicPolicy=True,
BlockPublicAcls=True,
RestrictPublicBuckets=True,
),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.
ServerSideEncryptionByDefault(
SSEAlgorithm="AES256"))
]),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
BucketName=bucket_name,
VersioningConfiguration=s3.VersioningConfiguration(
Status="Enabled"),
))
source_stage.Actions.append(
codepipeline.Actions(
RunOrder=1,
ActionTypeId=codepipeline.ActionTypeId(
Category="Source",
Owner="AWS",
Version="1",
Provider="S3",
),
OutputArtifacts=[codepipeline.OutputArtifacts(Name="Source")],
Configuration={
"S3Bucket":
bucket_name,
"S3ObjectKey":
source.get("Configuration").get("S3ObjectKey"),
"PollForSourceChanges":
source.get("Configuration").get("PollForSourceChanges"),
},
Name="Source",
))
single_account_run_project_build_spec = dict(
version=0.2,
phases=dict(
install=install_spec,
build={
"commands": [
'echo "single_account: \\"${SINGLE_ACCOUNT_ID}\\"" > parameters.yaml',
"cat parameters.yaml",
"zip parameters.zip parameters.yaml",
"aws s3 cp parameters.zip s3://sc-puppet-parameterised-runs-${PUPPET_ACCOUNT_ID}/parameters.zip",
]
},
post_build={
"commands": [
"servicecatalog-puppet wait-for-parameterised-run-to-complete",
]
},
),
artifacts=dict(
name="DeployProject",
files=[
"ServiceCatalogPuppet/manifest.yaml",
"ServiceCatalogPuppet/manifest-expanded.yaml",
"results/*/*",
"output/*/*",
"exploded_results/*/*",
"tasks.log",
],
),
)
single_account_run_project_args = dict(
Name="servicecatalog-puppet-single-account-run",
Description="Runs puppet for a single account - SINGLE_ACCOUNT_ID",
ServiceRole=t.GetAtt(deploy_role, "Arn"),
Tags=t.Tags.from_dict(**{"ServiceCatalogPuppet:Actor": "Framework"}),
Artifacts=codebuild.Artifacts(Type="NO_ARTIFACTS", ),
TimeoutInMinutes=480,
Environment=codebuild.Environment(
ComputeType=t.Ref(deploy_environment_compute_type_parameter),
Image="aws/codebuild/standard:4.0",
Type="LINUX_CONTAINER",
EnvironmentVariables=[
{
"Type": "PLAINTEXT",
"Name": "SINGLE_ACCOUNT_ID",
"Value": "CHANGE_ME",
},
] + deploy_env_vars,
),
Source=codebuild.Source(
Type="NO_SOURCE",
BuildSpec=yaml.safe_dump(single_account_run_project_build_spec),
),
)
single_account_run_project = template.add_resource(
codebuild.Project("SingleAccountRunProject",
**single_account_run_project_args))
single_account_run_project_build_spec["phases"]["post_build"]["commands"] = [
"servicecatalog-puppet wait-for-parameterised-run-to-complete --on-complete-url $CALLBACK_URL"
]
single_account_run_project_args[
"Name"] = "servicecatalog-puppet-single-account-run-with-callback"
single_account_run_project_args[
"Description"] = "Runs puppet for a single account - SINGLE_ACCOUNT_ID and then does a http put"
single_account_run_project_args.get(
"Environment").EnvironmentVariables.append({
"Type": "PLAINTEXT",
"Name": "CALLBACK_URL",
"Value": "CHANGE_ME",
})
single_account_run_project_args["Source"] = codebuild.Source(
Type="NO_SOURCE",
BuildSpec=yaml.safe_dump(single_account_run_project_build_spec),
)
single_account_run_project_with_callback = template.add_resource(
codebuild.Project("SingleAccountRunWithCallbackProject",
**single_account_run_project_args))
stages = [source_stage]
if should_validate:
template.add_resource(
codebuild.Project(
"ValidateProject",
Name="servicecatalog-puppet-validate",
ServiceRole=t.GetAtt("DeployRole", "Arn"),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
Artifacts=codebuild.Artifacts(Type="CODEPIPELINE"),
TimeoutInMinutes=60,
Environment=codebuild.Environment(
ComputeType="BUILD_GENERAL1_SMALL",
Image="aws/codebuild/standard:4.0",
Type="LINUX_CONTAINER",
),
Source=codebuild.Source(
BuildSpec=yaml.safe_dump(
dict(
version="0.2",
phases={
"install": {
"runtime-versions": {
"python": "3.7",
},
"commands": [
f"pip install {puppet_version}"
if "http" in puppet_version else
f"pip install aws-service-catalog-puppet=={puppet_version}",
],
},
"build": {
"commands": [
"servicecatalog-puppet validate manifest.yaml"
]
},
},
)),
Type="CODEPIPELINE",
),
Description="Validate the manifest.yaml file",
))
stages.append(
codepipeline.Stages(
Name="Validate",
Actions=[
codepipeline.Actions(
InputArtifacts=[
codepipeline.InputArtifacts(Name="Source"),
],
Name="Validate",
ActionTypeId=codepipeline.ActionTypeId(
Category="Build",
Owner="AWS",
Version="1",
Provider="CodeBuild",
),
OutputArtifacts=[
codepipeline.OutputArtifacts(
Name="ValidateProject")
],
Configuration={
"ProjectName": t.Ref("ValidateProject"),
"PrimarySource": "Source",
},
RunOrder=1,
),
],
))
if is_manual_approvals:
deploy_stage = codepipeline.Stages(
Name="Deploy",
Actions=[
codepipeline.Actions(
InputArtifacts=[
codepipeline.InputArtifacts(Name="Source"),
codepipeline.InputArtifacts(
Name="ParameterisedSource"),
],
Name="DryRun",
ActionTypeId=codepipeline.ActionTypeId(
Category="Build",
Owner="AWS",
Version="1",
Provider="CodeBuild",
),
OutputArtifacts=[
codepipeline.OutputArtifacts(Name="DryRunProject")
],
Configuration={
"ProjectName": t.Ref("DryRunProject"),
"PrimarySource": "Source",
},
RunOrder=1,
),
codepipeline.Actions(
ActionTypeId=codepipeline.ActionTypeId(
Category="Approval",
Owner="AWS",
Version="1",
Provider="Manual",
),
Configuration={
"NotificationArn":
t.Ref("DryRunNotificationTopic"),
"CustomData":
"Approve when you are happy with the dry run.",
},
Name="DryRunApproval",
RunOrder=2,
),
codepipeline.Actions(
InputArtifacts=[
codepipeline.InputArtifacts(Name="Source"),
codepipeline.InputArtifacts(
Name="ParameterisedSource"),
],
Name="Deploy",
ActionTypeId=codepipeline.ActionTypeId(
Category="Build",
Owner="AWS",
Version="1",
Provider="CodeBuild",
),
OutputArtifacts=[
codepipeline.OutputArtifacts(Name="DeployProject")
],
Configuration={
"ProjectName": t.Ref("DeployProject"),
"PrimarySource": "Source",
},
RunOrder=3,
),
],
)
else:
deploy_stage = codepipeline.Stages(
Name="Deploy",
Actions=[
codepipeline.Actions(
InputArtifacts=[
codepipeline.InputArtifacts(Name="Source"),
codepipeline.InputArtifacts(
Name="ParameterisedSource"),
],
Name="Deploy",
ActionTypeId=codepipeline.ActionTypeId(
Category="Build",
Owner="AWS",
Version="1",
Provider="CodeBuild",
),
OutputArtifacts=[
codepipeline.OutputArtifacts(Name="DeployProject")
],
Configuration={
"ProjectName":
t.Ref("DeployProject"),
"PrimarySource":
"Source",
"EnvironmentVariables":
'[{"name":"EXECUTION_ID","value":"#{codepipeline.PipelineExecutionId}","type":"PLAINTEXT"}]',
},
RunOrder=1,
),
],
)
stages.append(deploy_stage)
pipeline = template.add_resource(
codepipeline.Pipeline(
"Pipeline",
RoleArn=t.GetAtt("PipelineRole", "Arn"),
Stages=stages,
Name=t.Sub("${AWS::StackName}-pipeline"),
ArtifactStore=codepipeline.ArtifactStore(
Type="S3",
Location=t.Sub(
"sc-puppet-pipeline-artifacts-${AWS::AccountId}-${AWS::Region}"
),
),
RestartExecutionOnUpdate=True,
))
if is_github:
template.add_resource(
codepipeline.Webhook(
"Webhook",
AuthenticationConfiguration=codepipeline.
WebhookAuthConfiguration(SecretToken=t.Join(
"",
[
"{{resolve:secretsmanager:",
source.get("Configuration").get(
"SecretsManagerSecret"),
":SecretString:SecretToken}}",
],
)),
Filters=[
codepipeline.WebhookFilterRule(
JsonPath="$.ref",
MatchEquals="refs/heads/" +
source.get("Configuration").get("Branch"),
)
],
Authentication="GITHUB_HMAC",
TargetPipeline=t.Ref(pipeline),
TargetAction="Source",
Name=t.Sub("${AWS::StackName}-webhook"),
TargetPipelineVersion=t.GetAtt(pipeline, "Version"),
RegisterWithThirdParty="true",
))
deploy_project_build_spec = dict(
version=0.2,
phases=dict(
install={
"runtime-versions":
dict(python="3.7"),
"commands": [
f"pip install {puppet_version}"
if "http" in puppet_version else
f"pip install aws-service-catalog-puppet=={puppet_version}",
],
},
pre_build={
"commands": [
"servicecatalog-puppet --info expand --parameter-override-file $CODEBUILD_SRC_DIR_ParameterisedSource/parameters.yaml manifest.yaml",
]
},
build={
"commands": [
"servicecatalog-puppet --info deploy --num-workers ${NUM_WORKERS} manifest-expanded.yaml",
]
},
),
artifacts=dict(
name="DeployProject",
files=[
"manifest-expanded.yaml",
"results/*/*",
"output/*/*",
"exploded_results/*/*",
"tasks.log",
],
),
)
deploy_project_args = dict(
Name="servicecatalog-puppet-deploy",
ServiceRole=t.GetAtt(deploy_role, "Arn"),
Tags=t.Tags.from_dict(**{"ServiceCatalogPuppet:Actor": "Framework"}),
Artifacts=codebuild.Artifacts(Type="CODEPIPELINE", ),
TimeoutInMinutes=480,
Environment=codebuild.Environment(
ComputeType=t.Ref(deploy_environment_compute_type_parameter),
Image="aws/codebuild/standard:4.0",
Type="LINUX_CONTAINER",
EnvironmentVariables=[
{
"Type": "PARAMETER_STORE",
"Name": "NUM_WORKERS",
"Value": t.Ref(num_workers_ssm_parameter),
},
{
"Type":
"PARAMETER_STORE",
"Name":
"SPOKE_EXECUTION_MODE_DEPLOY_ENV",
"Value":
constants.SPOKE_EXECUTION_MODE_DEPLOY_ENV_PARAMETER_NAME,
},
] + deploy_env_vars,
),
Source=codebuild.Source(
Type="CODEPIPELINE",
BuildSpec=yaml.safe_dump(deploy_project_build_spec),
),
Description="deploys out the products to be deployed",
)
deploy_project = template.add_resource(
codebuild.Project("DeployProject", **deploy_project_args))
deploy_project_build_spec["phases"]["build"]["commands"] = [
"servicecatalog-puppet --info dry-run manifest-expanded.yaml"
]
deploy_project_build_spec["artifacts"]["name"] = "DryRunProject"
deploy_project_args["Name"] = "servicecatalog-puppet-dryrun"
deploy_project_args[
"Description"] = "dry run of servicecatalog-puppet-dryrun"
deploy_project_args["Source"] = codebuild.Source(
Type="CODEPIPELINE",
BuildSpec=yaml.safe_dump(deploy_project_build_spec),
)
dry_run_project = template.add_resource(
codebuild.Project("DryRunProject", **deploy_project_args))
bootstrap_project = template.add_resource(
codebuild.Project(
"BootstrapProject",
Name="servicecatalog-puppet-bootstrap-spokes-in-ou",
ServiceRole=t.GetAtt("DeployRole", "Arn"),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
Artifacts=codebuild.Artifacts(Type="NO_ARTIFACTS"),
TimeoutInMinutes=60,
Environment=codebuild.Environment(
ComputeType="BUILD_GENERAL1_SMALL",
Image="aws/codebuild/standard:4.0",
Type="LINUX_CONTAINER",
EnvironmentVariables=[
{
"Type": "PLAINTEXT",
"Name": "OU_OR_PATH",
"Value": "CHANGE_ME"
},
{
"Type": "PLAINTEXT",
"Name": "IAM_ROLE_NAME",
"Value": "OrganizationAccountAccessRole",
},
{
"Type": "PLAINTEXT",
"Name": "IAM_ROLE_ARNS",
"Value": ""
},
],
),
Source=codebuild.Source(
BuildSpec=
"version: 0.2\nphases:\n install:\n runtime-versions:\n python: 3.7\n commands:\n - pip install aws-service-catalog-puppet\n build:\n commands:\n - servicecatalog-puppet bootstrap-spokes-in-ou $OU_OR_PATH $IAM_ROLE_NAME $IAM_ROLE_ARNS\nartifacts:\n files:\n - results/*/*\n - output/*/*\n name: BootstrapProject\n",
Type="NO_SOURCE",
),
Description="Bootstrap all the accounts in an OU",
))
template.add_resource(
codebuild.Project(
"BootstrapASpokeProject",
Name="servicecatalog-puppet-bootstrap-spoke",
ServiceRole=t.GetAtt("DeployRole", "Arn"),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
Artifacts=codebuild.Artifacts(Type="NO_ARTIFACTS"),
TimeoutInMinutes=60,
Environment=codebuild.Environment(
ComputeType="BUILD_GENERAL1_SMALL",
Image="aws/codebuild/standard:4.0",
Type="LINUX_CONTAINER",
EnvironmentVariables=[
{
"Type": "PLAINTEXT",
"Name": "PUPPET_ACCOUNT_ID",
"Value": t.Sub("${AWS::AccountId}"),
},
{
"Type": "PLAINTEXT",
"Name": "ORGANIZATION_ACCOUNT_ACCESS_ROLE_ARN",
"Value": "CHANGE_ME",
},
{
"Type": "PLAINTEXT",
"Name": "ASSUMABLE_ROLE_IN_ROOT_ACCOUNT",
"Value": "CHANGE_ME",
},
],
),
Source=codebuild.Source(
BuildSpec=yaml.safe_dump(
dict(
version=0.2,
phases=dict(
install=install_spec,
build={
"commands": [
"servicecatalog-puppet bootstrap-spoke-as ${PUPPET_ACCOUNT_ID} ${ASSUMABLE_ROLE_IN_ROOT_ACCOUNT} ${ORGANIZATION_ACCOUNT_ACCESS_ROLE_ARN}"
]
},
),
)),
Type="NO_SOURCE",
),
Description="Bootstrap given account as a spoke",
))
cloud_formation_events_queue = template.add_resource(
sqs.Queue(
"CloudFormationEventsQueue",
QueueName="servicecatalog-puppet-cloudformation-events",
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
))
cloud_formation_events_queue_policy = template.add_resource(
sqs.QueuePolicy(
"CloudFormationEventsQueuePolicy",
Queues=[t.Ref(cloud_formation_events_queue)],
PolicyDocument={
"Id":
"AllowSNS",
"Version":
"2012-10-17",
"Statement": [{
"Sid": "allow-send-message",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": ["sqs:SendMessage"],
"Resource": "*",
"Condition": {
"ArnEquals": {
"aws:SourceArn":
t.Sub(
"arn:${AWS::Partition}:sns:*:${AWS::AccountId}:servicecatalog-puppet-cloudformation-regional-events"
)
}
},
}],
},
))
spoke_deploy_bucket = template.add_resource(
s3.Bucket(
"SpokeDeployBucket",
PublicAccessBlockConfiguration=s3.PublicAccessBlockConfiguration(
IgnorePublicAcls=True,
BlockPublicPolicy=True,
BlockPublicAcls=True,
RestrictPublicBuckets=True,
),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.
ServerSideEncryptionByDefault(SSEAlgorithm="AES256"))
]),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
BucketName=t.Sub("sc-puppet-spoke-deploy-${AWS::AccountId}"),
VersioningConfiguration=s3.VersioningConfiguration(
Status="Enabled"),
))
caching_bucket = template.add_resource(
s3.Bucket(
"CachingBucket",
PublicAccessBlockConfiguration=s3.PublicAccessBlockConfiguration(
BlockPublicAcls=True,
BlockPublicPolicy=True,
IgnorePublicAcls=True,
RestrictPublicBuckets=True,
),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.
ServerSideEncryptionByDefault(SSEAlgorithm="AES256"))
]),
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
BucketName=t.Sub(
"sc-puppet-caching-bucket-${AWS::AccountId}-${AWS::Region}"),
VersioningConfiguration=s3.VersioningConfiguration(
Status="Enabled"),
))
template.add_output(
t.Output(
"CloudFormationEventsQueueArn",
Value=t.GetAtt(cloud_formation_events_queue, "Arn"),
))
template.add_output(t.Output("Version", Value=t.GetAtt(param, "Value")))
template.add_output(
t.Output("ManualApprovalsParam",
Value=t.GetAtt(manual_approvals_param, "Value")))
template.add_resource(
ssm.Parameter(
"DefaultTerraformVersion",
Type="String",
Name=constants.DEFAULT_TERRAFORM_VERSION_PARAMETER_NAME,
Value=constants.DEFAULT_TERRAFORM_VERSION_VALUE,
))
return template
def generate_cft():
VPC_NETWORK = "10.0.0.0/16"
VPC_PRIVATE = "10.0.0.0/24"
t = troposphere.Template()
t.add_description("HaaS Stack")
key_name = t.add_parameter(
troposphere.Parameter(
"KeyName",
Type="AWS::EC2::KeyPair::KeyName",
ConstraintDescription="must be a valid keypair Id",
))
username_and_password = t.add_parameter(
troposphere.Parameter(
"UserNameAndPassword",
Type="String",
Default="",
Description=
"(Optional) Enter like: username/password Used to log into ECL Watch and ECL IDE."
))
cluster_size = t.add_parameter(
troposphere.Parameter(
"ClusterSize",
Type="Number",
Default="1",
Description="Number of slave instances to be launched"))
num_slaves = t.add_parameter(
troposphere.Parameter(
"NumberOfSlavesPerNode",
Type="Number",
Default="1",
Description="Number of THOR slave nodes per slave instance"))
master_instance_type = t.add_parameter(
troposphere.Parameter(
"MasterInstanceType",
Type="String",
AllowedValues=[
't2.micro', 'c4.large', 'c4.xlarge', 'c4.2xlarge', 'm4.large',
'm4.xlarge', 'm4.2xlarge', 'r4.large', 'r4.xlarge',
'r4.2xlarge'
],
Default="c4.large",
Description="HPCC Thor Master EC2 instance type"))
slave_instance_type = t.add_parameter(
troposphere.Parameter("SlaveInstanceType",
Type="String",
AllowedValues=[
't2.micro', 'c4.large', 'c4.xlarge',
'c4.2xlarge', 'm4.large', 'm4.xlarge',
'm4.2xlarge', 'r4.large', 'r4.xlarge',
'r4.2xlarge'
],
Default="c4.large",
Description="HPCC Thor Slave EC2 instance type"))
vpc_availability_zone = t.add_parameter(
troposphere.Parameter(
"AvailabilityZone",
Type="String",
AllowedValues=['us-east-1d'],
Default="us-east-1d",
Description="Availability zone",
))
t.add_mapping('RegionMap', {'us-east-1': {'64': 'ami-24c2ee32'}})
instance_role = t.add_resource(
troposphere.iam.Role(
"HPCCInstanceRoles",
AssumeRolePolicyDocument=awacs.aws.Policy(Statement=[
awacs.aws.Statement(Effect=awacs.aws.Allow,
Action=[awacs.sts.AssumeRole],
Principal=awacs.aws.Principal(
"Service", ["ec2.amazonaws.com"]))
]),
Policies=[
troposphere.iam.Policy(
PolicyName="root",
PolicyDocument=awacs.aws.Policy(Statement=[
awacs.aws.Statement(Effect=awacs.aws.Allow,
Action=[awacs.aws.Action("*")],
Resource=["*"])
]))
],
Path="/"))
instance_profile = t.add_resource(
troposphere.iam.InstanceProfile("HPCCInstanceProfile",
Path="/",
Roles=[troposphere.Ref(instance_role)
]))
vpc = t.add_resource(
troposphere.ec2.VPC(
"HPCCVpc",
CidrBlock=VPC_NETWORK,
InstanceTenancy="default",
EnableDnsSupport=True,
EnableDnsHostnames=False,
Tags=troposphere.Tags(Name=troposphere.Ref("AWS::StackName"))))
internetGateway = t.add_resource(
troposphere.ec2.InternetGateway(
"InternetGateway",
Tags=troposphere.Tags(Name=troposphere.Join(
"-", [troposphere.Ref("AWS::StackName"), "gateway"]), ),
))
gatewayAttachment = t.add_resource(
troposphere.ec2.VPCGatewayAttachment(
"InternetGatewayAttachment",
InternetGatewayId=troposphere.Ref(internetGateway),
VpcId=troposphere.Ref(vpc)))
# public routing table
publicRouteTable = t.add_resource(
troposphere.ec2.RouteTable(
"PublicRouteTable",
VpcId=troposphere.Ref(vpc),
Tags=troposphere.Tags(Name=troposphere.Join(
"-", [troposphere.Ref("AWS::StackName"), "public-rt"]), ),
))
internetRoute = t.add_resource(
troposphere.ec2.Route("RouteToInternet",
DestinationCidrBlock="0.0.0.0/0",
GatewayId=troposphere.Ref(internetGateway),
RouteTableId=troposphere.Ref(publicRouteTable),
DependsOn=gatewayAttachment.title))
subnet = t.add_resource(
troposphere.ec2.Subnet(
"Subnet",
CidrBlock=VPC_PRIVATE,
Tags=troposphere.Tags(Name=troposphere.Join(
"-", [troposphere.Ref("AWS::StackName"), "subnet"]), ),
VpcId=troposphere.Ref(vpc)))
t.add_resource(
troposphere.ec2.SubnetRouteTableAssociation(
"SubnetRouteTableAssociation",
RouteTableId=troposphere.Ref(publicRouteTable),
SubnetId=troposphere.Ref(subnet)))
placement_group = t.add_resource(
troposphere.ec2.PlacementGroup("HPCCPlacementGroup",
Strategy="cluster"))
security_groups = t.add_resource(
troposphere.ec2.SecurityGroup(
"HPCCSecurityGroups",
GroupDescription="Enable SSH and HTTP access on the inbound port",
SecurityGroupEgress=[
troposphere.ec2.SecurityGroupRule(
IpProtocol="-1",
CidrIp="0.0.0.0/0",
),
],
SecurityGroupIngress=[
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8888,
ToPort=8888,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=9042,
ToPort=9042,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=7000,
ToPort=7000,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=7001,
ToPort=7001,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=7199,
ToPort=7199,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=9160,
ToPort=9160,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=61620,
ToPort=61620,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=61621,
ToPort=61621,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8002,
ToPort=8002,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8010,
ToPort=8010,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8015,
ToPort=8015,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8145,
ToPort=8145,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=22,
ToPort=22,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=0,
ToPort=65535,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="udp",
FromPort=0,
ToPort=65535,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8050,
ToPort=8050,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=8008,
ToPort=8008,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="tcp",
FromPort=9876,
ToPort=9876,
CidrIp="0.0.0.0/0",
),
troposphere.ec2.SecurityGroupRule(
IpProtocol="icmp",
FromPort=-1,
ToPort=-1,
CidrIp="0.0.0.0/0",
),
],
VpcId=troposphere.Ref(vpc)))
# AutoScaling
slave_launch_config = t.add_resource(
troposphere.autoscaling.LaunchConfiguration(
"SlaveLaunchCfg",
ImageId=troposphere.FindInMap("RegionMap",
troposphere.Ref("AWS::Region"),
"64"),
InstanceType=troposphere.Ref(slave_instance_type),
AssociatePublicIpAddress="True",
KeyName=troposphere.Ref(key_name),
SecurityGroups=[troposphere.Ref(security_groups)],
IamInstanceProfile=troposphere.Ref(instance_profile),
UserData=troposphere.Base64(
troposphere.Join('\n', [
"#!/bin/bash",
"exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1",
"echo [Initialization] starting the slave node",
troposphere.Join(" ", [
"su - osr /bin/bash -c 'cd /home/osr/project-aws; git pull; /bin/bash scripts/auto_hpcc.sh",
troposphere.Ref("AWS::StackName"),
troposphere.Ref("AWS::Region"),
"'",
]), "echo [Initialization] completed the slave node",
"echo SCRIPT: 'Signal stack that setup of HPCC System is complete.'",
troposphere.Join(" ", [
"/usr/local/bin/cfn-signal -e 0 --stack ",
troposphere.Ref("AWS::StackName"),
"--resource SlaveASG ", "--region ",
troposphere.Ref("AWS::Region")
]),
"echo SCRIPT: 'Done signaling stack that setup of HPCC System has completed.'"
])),
))
slave_autoscaling_group = t.add_resource(
troposphere.autoscaling.AutoScalingGroup(
"SlaveASG",
DesiredCapacity=troposphere.Ref(cluster_size),
# @TODO: disable here to support t2.micro for cheap testing
#PlacementGroup=troposphere.Ref(placement_group),
LaunchConfigurationName=troposphere.Ref(slave_launch_config),
MinSize=troposphere.Ref(cluster_size),
MaxSize=troposphere.Ref(cluster_size),
HealthCheckType="EC2",
HealthCheckGracePeriod="300",
VPCZoneIdentifier=[troposphere.Ref(subnet)],
#AvailabilityZones=[troposphere.Ref(vpc_availability_zone)],
Tags=[
troposphere.autoscaling.Tag("StackName",
troposphere.Ref("AWS::StackName"),
True),
troposphere.autoscaling.Tag("slavesPerNode",
troposphere.Ref(num_slaves), True),
troposphere.autoscaling.Tag(
"UserNameAndPassword",
troposphere.Ref(username_and_password), True),
troposphere.autoscaling.Tag(
"Name",
troposphere.Join(
"-", [troposphere.Ref("AWS::StackName"), "Slave"]),
True),
],
))
master_launch_config = t.add_resource(
troposphere.autoscaling.LaunchConfiguration(
"MasterLaunchCfg",
ImageId=troposphere.FindInMap("RegionMap",
troposphere.Ref("AWS::Region"),
"64"),
InstanceType=troposphere.Ref(master_instance_type),
AssociatePublicIpAddress="True",
KeyName=troposphere.Ref(key_name),
SecurityGroups=[troposphere.Ref(security_groups)],
IamInstanceProfile=troposphere.Ref(instance_profile),
UserData=troposphere.Base64(
troposphere.Join('\n', [
"#!/bin/bash",
"exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1",
"echo [Initialization] starting the master node",
troposphere.Join(" ", [
"su - osr /bin/bash -c 'cd /home/osr/project-aws; git pull; /bin/bash scripts/auto_hpcc.sh",
troposphere.Ref("AWS::StackName"),
troposphere.Ref("AWS::Region"),
"'",
]), "echo [Initialization] completed the master node",
"echo SCRIPT: 'Signal stack that setup of HPCC System is complete.'",
troposphere.Join(" ", [
"/usr/local/bin/cfn-signal -e 0 --stack ",
troposphere.Ref("AWS::StackName"),
"--resource MasterASG ", "--region ",
troposphere.Ref("AWS::Region")
]),
"echo SCRIPT: 'Done signaling stack that setup of HPCC System has completed.'"
])),
))
master_autoscaling_group = t.add_resource(
troposphere.autoscaling.AutoScalingGroup(
"MasterASG",
DesiredCapacity="1", # need to update x -> N+x
# @TODO: disable here to support t2.micro for cheap testing
#PlacementGroup=troposphere.Ref(placement_group),
LaunchConfigurationName=troposphere.Ref(master_launch_config),
MinSize="1",
MaxSize="1",
HealthCheckType="EC2",
HealthCheckGracePeriod="300",
VPCZoneIdentifier=[troposphere.Ref(subnet)],
#AvailabilityZones=[troposphere.Ref(vpc_availability_zone)],
Tags=[
troposphere.autoscaling.Tag("StackName",
troposphere.Ref("AWS::StackName"),
True),
troposphere.autoscaling.Tag("slavesPerNode",
troposphere.Ref(num_slaves), True),
troposphere.autoscaling.Tag(
"UserNameAndPassword",
troposphere.Ref(username_and_password), True),
troposphere.autoscaling.Tag(
"Name",
troposphere.Join(
"-", [troposphere.Ref("AWS::StackName"), "Master"]),
True),
],
))
print(t.to_json())
return t.to_dict()
def __init__(
self,
paco_ctx,
account_ctx,
aws_region,
stack_group,
stack_tags,
resource,
):
# Route53 metrics only go to us-east-1
# save the app region so it can be used in Name and Tags
self.app_aws_region = aws_region
aws_region = 'us-east-1'
self.health_check = resource
self.alarm_action_param_map = {}
self.notification_param_map = {}
super().__init__(
paco_ctx,
account_ctx,
aws_region,
config_ref=self.health_check.paco_ref_parts,
stack_group=stack_group,
stack_tags=stack_tags,
enabled=self.health_check.is_enabled(),
)
self.set_aws_name('Route53HealthCheck', self.health_check.name)
# Troposphere Template Initialization
self.init_template('Route 53 health check')
# Health check Resource
if self.health_check.is_enabled():
health_check_logical_id = self.create_cfn_logical_id(
'Route53HealthCheck' + self.health_check.name)
cfn_export_dict = {}
cfn_export_dict[
'HealthCheckConfig'] = self.health_check.cfn_export_dict
if self.health_check.ip_address != None:
# Set the IPAddress to ping
ip_address_param = self.create_cfn_parameter(
param_type='String',
name='IPAddress',
description='IP address to monitor.',
value=self.health_check.ip_address + '.address',
)
cfn_export_dict['HealthCheckConfig'][
'IPAddress'] = troposphere.Ref(ip_address_param)
else:
# FullyQualifiedDomainName can be either a domain_name or a ref to an ALB endpoint
if self.health_check.domain_name != None:
fqdn_value = self.health_check.domain_name
else:
fqdn_value = self.health_check.load_balancer + '.dnsname'
fqdn_param = self.create_cfn_parameter(
param_type='String',
name='FQDNEndpoint',
description=
'Fully-qualified domain name of the endpoint to monitor.',
value=fqdn_value,
)
cfn_export_dict['HealthCheckConfig'][
'FullyQualifiedDomainName'] = troposphere.Ref(fqdn_param)
# Set the Name in the HealthCheckTags
# Route53 is global, but we add the app's region in the name
cfn_export_dict['HealthCheckTags'] = troposphere.Tags(
Name=self.aws_name + '-' + self.app_aws_region)
health_check_resource = troposphere.route53.HealthCheck.from_dict(
health_check_logical_id, cfn_export_dict)
self.template.add_resource(health_check_resource)
# CloudWatch Alarm
# ToDo: allow configurtion of this alarm from the model
alarm = CloudWatchAlarm('HealthCheckAlarm', self.health_check)
alarm.overrode_region_name = 'us-east-1'
alarm.metric_name = "HealthCheckPercentageHealthy"
alarm.classification = 'health'
alarm.severity = 'critical'
alarm.namespace = "AWS/Route53"
alarm.period = 60
alarm.evaluation_periods = 1
alarm.threshold = 18.0 # As recommended by AWS
alarm.comparison_operator = 'LessThanOrEqualToThreshold'
alarm.statistic = "Average"
alarm.treat_missing_data = 'breaching'
cfn_export_dict = alarm.cfn_export_dict
cfn_export_dict['Dimensions'] = [{
'Name':
'HealthCheckId',
'Value':
troposphere.Ref(health_check_resource),
}]
cfn_export_dict['Namespace'] = "AWS/Route53"
notification_cfn_refs = self.create_notification_params(alarm)
cfn_export_dict['AlarmDescription'] = alarm.get_alarm_description(
notification_cfn_refs)
self.set_alarm_actions_to_cfn_export(alarm, cfn_export_dict)
alarm_resource = troposphere.cloudwatch.Alarm.from_dict(
'HealthCheckAlarm', cfn_export_dict)
alarm_resource.DependsOn = health_check_logical_id
self.template.add_resource(alarm_resource)
# Generate the Template
self.set_template()
class VPC:
def __init__(self, Name, CidrBlock):
self.name = Name
self.network = netaddr.IPNetwork(CidrBlock)
parser = argparse.ArgumentParser()
parser.add_argument("yaml_file")
args = parser.parse_args()
stack_id = tr.Ref('AWS::StackId')
region = tr.Ref('AWS::Region')
stack_name = tr.Ref('AWS::StackName')
app_tag = tr.Tags(Application=stack_id)
# object representing CloudFormation template
t = tr.Template()
t.add_version('2010-09-09')
t.add_description('CloudFormation template for VPC and Subnets')
# yaml file as dict
try:
with open(args.yaml_file) as f:
yaml_file = yaml.load(f)
except Exception as e:
print('Failed to open yaml: %s' % e)
sys.exit(1)
# vpc:
def run(self):
puppet_version = constants.VERSION
description = f"""Bootstrap template used to configure spoke account for terraform use
{{"version": "{puppet_version}", "framework": "servicecatalog-puppet", "role": "bootstrap-spoke-terraform"}}"""
service_role = t.Sub(
"arn:aws:iam::${AWS::AccountId}:role/servicecatalog-puppet/PuppetDeployInSpokeRole"
)
template = t.Template(Description=description)
state = template.add_resource(
s3.Bucket(
"state",
BucketName=t.Sub("sc-puppet-state-${AWS::AccountId}"),
VersioningConfiguration=s3.VersioningConfiguration(
Status="Enabled"),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.
ServerSideEncryptionByDefault(
SSEAlgorithm="AES256"))
]),
PublicAccessBlockConfiguration=s3.
PublicAccessBlockConfiguration(
BlockPublicAcls=True,
BlockPublicPolicy=True,
IgnorePublicAcls=True,
RestrictPublicBuckets=True,
),
Tags=t.Tags({"ServiceCatalogPuppet:Actor": "Framework"}),
))
template.add_resource(
s3.BucketPolicy(
"statePolicy",
Bucket=t.Ref(state),
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": [
"s3:GetObject*",
"s3:PutObject*",
],
"Principal": {
"AWS": self.puppet_account_id
},
"Resource": t.Join("/",
[t.GetAtt(state, "Arn"), "*"]),
"Effect": "Allow",
"Sid": "AllowPuppet",
},
],
},
))
execute_build_spec = dict(
version="0.2",
phases=dict(
install=dict(commands=[
"mkdir -p /root/downloads",
"curl -s -qL -o /root/downloads/terraform_${TERRAFORM_VERSION}_linux_amd64.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip",
"unzip /root/downloads/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/bin/",
"chmod +x /usr/bin/terraform",
"terraform --version",
"aws s3 cp $ZIP source.zip",
"unzip source.zip",
], ),
pre_build=dict(commands=[
"aws s3 cp $STATE_FILE terraform.tfstate || echo 'no statefile copied'",
'ASSUME_ROLE_ARN="arn:aws:iam::${TARGET_ACCOUNT}:role/servicecatalog-puppet/PuppetRole"',
"TEMP_ROLE=$(aws sts assume-role --role-arn $ASSUME_ROLE_ARN --role-session-name terraform)",
"export TEMP_ROLE",
'export AWS_ACCESS_KEY_ID=$(echo "${TEMP_ROLE}" | jq -r ".Credentials.AccessKeyId")',
'export AWS_SECRET_ACCESS_KEY=$(echo "${TEMP_ROLE}" | jq -r ".Credentials.SecretAccessKey")',
'export AWS_SESSION_TOKEN=$(echo "${TEMP_ROLE}" | jq -r ".Credentials.SessionToken")',
"aws sts get-caller-identity",
"terraform init",
], ),
build=dict(commands=[
"terraform apply -auto-approve",
]),
post_build=dict(commands=[
"terraform output -json > outputs.json",
"unset AWS_ACCESS_KEY_ID",
"unset AWS_SECRET_ACCESS_KEY",
"unset AWS_SESSION_TOKEN",
"aws sts get-caller-identity",
"aws s3 cp terraform.tfstate $STATE_FILE",
]),
),
artifacts=dict(files=[
"outputs.json",
], ),
)
execute_terraform = dict(
Name=constants.EXECUTE_TERRAFORM_PROJECT_NAME,
ServiceRole=service_role,
Tags=t.Tags.from_dict(
**{"ServiceCatalogPuppet:Actor": "Framework"}),
Artifacts=codebuild.Artifacts(
Type="S3",
Location=t.Ref("state"),
Path="terraform-executions",
Name="artifacts-execute",
NamespaceType="BUILD_ID",
),
TimeoutInMinutes=480,
Environment=codebuild.Environment(
ComputeType="BUILD_GENERAL1_SMALL",
Image=constants.CODEBUILD_DEFAULT_IMAGE,
Type="LINUX_CONTAINER",
EnvironmentVariables=[
codebuild.EnvironmentVariable(
Name="TERRAFORM_VERSION",
Type="PARAMETER_STORE",
Value=constants.
DEFAULT_TERRAFORM_VERSION_PARAMETER_NAME,
),
] + [
codebuild.EnvironmentVariable(
Name=name,
Type="PLAINTEXT",
Value="CHANGE_ME",
) for name in ["TARGET_ACCOUNT", "ZIP", "STATE_FILE"]
],
),
Source=codebuild.Source(
BuildSpec=yaml.safe_dump(execute_build_spec),
Type="NO_SOURCE",
),
Description=
"Execute the given terraform in the given account using the given state file",
)
# execute
template.add_resource(
codebuild.Project("ExecuteTerraformProject", **execute_terraform))
# execute dry run
execute_dry_run_terraform = copy.deepcopy(execute_terraform)
execute_dry_run_terraform[
"Name"] = constants.EXECUTE_DRY_RUN_TERRAFORM_PROJECT_NAME
execute_dry_run_terraform["Description"] = execute_dry_run_terraform[
"Description"].replace("Execute", "DRY RUN of Execute")
execute_dry_run_build_spec = copy.deepcopy(execute_build_spec)
execute_dry_run_build_spec["phases"]["build"]["commands"] = [
"terraform plan -out=plan.bin",
"terraform show -json plan.bin > plan.json",
]
del execute_dry_run_build_spec["phases"]["post_build"]
execute_dry_run_build_spec["artifacts"] = dict(files=[
"plan.bin",
"plan.json",
], )
execute_dry_run_terraform["Source"] = codebuild.Source(
BuildSpec=yaml.safe_dump(execute_dry_run_build_spec),
Type="NO_SOURCE",
)
execute_dry_run_terraform["Artifacts"] = codebuild.Artifacts(
Type="S3",
Location=t.Ref("state"),
Path="terraform-executions",
Name="artifacts-execute-dry-run",
NamespaceType="BUILD_ID",
)
template.add_resource(
codebuild.Project("ExecuteDryRunTerraformProject",
**execute_dry_run_terraform))
# terminate
terminate_terraform = copy.deepcopy(execute_terraform)
terminate_terraform[
"Name"] = constants.TERMINATE_TERRAFORM_PROJECT_NAME
terminate_terraform["Description"] = terminate_terraform[
"Description"].replace("Execute", "Terminate")
terminate_build_spec = copy.deepcopy(execute_build_spec)
terminate_build_spec["phases"]["build"]["commands"] = [
"terraform destroy -auto-approve"
]
terminate_build_spec["phases"]["post_build"]["commands"] = [
"unset AWS_ACCESS_KEY_ID",
"unset AWS_SECRET_ACCESS_KEY",
"unset AWS_SESSION_TOKEN",
"aws sts get-caller-identity",
"aws s3 cp terraform.tfstate $STATE_FILE",
]
del terminate_build_spec["artifacts"]
terminate_terraform["Source"] = codebuild.Source(
BuildSpec=yaml.safe_dump(terminate_build_spec),
Type="NO_SOURCE",
)
terminate_terraform["Artifacts"] = codebuild.Artifacts(
Type="S3",
Location=t.Ref("state"),
Path="terraform-executions",
Name="artifacts-terminate",
NamespaceType="BUILD_ID",
)
template.add_resource(
codebuild.Project("TerminateTerraformProject",
**terminate_terraform))
# terminate dry run
termminate_dry_run_terraform = copy.deepcopy(execute_terraform)
termminate_dry_run_terraform[
"Name"] = constants.TERMINATE_DRY_RUN_TERRAFORM_PROJECT_NAME
new_description = termminate_dry_run_terraform["Description"].replace(
"Execute", "DRY RUN of Terminate")
termminate_dry_run_terraform["Description"] = new_description
termminate_dry_run_build_spec = copy.deepcopy(execute_build_spec)
termminate_dry_run_build_spec["phases"]["build"]["commands"] = [
"terraform plan -destroy -out=plan.bin",
"terraform show -json plan.bin > plan.json",
]
del termminate_dry_run_build_spec["phases"]["post_build"]
termminate_dry_run_build_spec["artifacts"] = dict(files=[
"plan.bin",
"plan.json",
], )
termminate_dry_run_terraform["Source"] = codebuild.Source(
BuildSpec=yaml.safe_dump(termminate_dry_run_build_spec),
Type="NO_SOURCE",
)
termminate_dry_run_terraform["Artifacts"] = codebuild.Artifacts(
Type="S3",
Location=t.Ref("state"),
Path="terraform-executions",
Name="artifacts-terminate-dry-run",
NamespaceType="BUILD_ID",
)
template.add_resource(
codebuild.Project("TerminateDryRunTerraformProject",
**termminate_dry_run_terraform))
self.write_output(template.to_yaml(), skip_json_dump=True)
def get_template(version: str, default_region_value) -> t.Template:
description = f"""Bootstrap template used to bootstrap a region of ServiceCatalog-Puppet master
{{"version": "{version}", "framework": "servicecatalog-puppet", "role": "bootstrap-master-region"}}"""
template = t.Template(Description=description)
version_parameter = template.add_parameter(
t.Parameter("Version", Default=version, Type="String")
)
default_region_value_parameter = template.add_parameter(
t.Parameter("DefaultRegionValue", Default=default_region_value, Type="String")
)
template.add_resource(
ssm.Parameter(
"DefaultRegionParam",
Name="/servicecatalog-puppet/home-region",
Type="String",
Value=t.Ref(default_region_value_parameter),
Tags={"ServiceCatalogPuppet:Actor": "Framework"},
)
)
version_ssm_parameter = template.add_resource(
ssm.Parameter(
"Param",
Name="service-catalog-puppet-regional-version",
Type="String",
Value=t.Ref(version_parameter),
Tags={"ServiceCatalogPuppet:Actor": "Framework"},
)
)
template.add_resource(
s3.Bucket(
"PipelineArtifactBucket",
BucketName=t.Sub(
"sc-puppet-pipeline-artifacts-${AWS::AccountId}-${AWS::Region}"
),
VersioningConfiguration=s3.VersioningConfiguration(Status="Enabled"),
BucketEncryption=s3.BucketEncryption(
ServerSideEncryptionConfiguration=[
s3.ServerSideEncryptionRule(
ServerSideEncryptionByDefault=s3.ServerSideEncryptionByDefault(
SSEAlgorithm="AES256"
)
)
]
),
PublicAccessBlockConfiguration=s3.PublicAccessBlockConfiguration(
BlockPublicAcls=True,
BlockPublicPolicy=True,
IgnorePublicAcls=True,
RestrictPublicBuckets=True,
),
Tags=t.Tags({"ServiceCatalogPuppet:Actor": "Framework"}),
)
)
regional_product_topic = template.add_resource(
sns.Topic(
"RegionalProductTopic",
DisplayName="servicecatalog-puppet-cloudformation-regional-events",
TopicName="servicecatalog-puppet-cloudformation-regional-events",
Subscription=[
sns.Subscription(
Endpoint=t.Sub(
"arn:${AWS::Partition}:sqs:${DefaultRegionValue}:${AWS::AccountId}:servicecatalog-puppet-cloudformation-events"
),
Protocol="sqs",
)
],
),
)
template.add_output(
t.Output("Version", Value=t.GetAtt(version_ssm_parameter, "Value"))
)
template.add_output(
t.Output("RegionalProductTopic", Value=t.Ref(regional_product_topic))
)
return template
