import pulumi
import pulumi_aws as aws
import pulumi_pulumi as pulumi
peer = pulumi.providers.Aws("peer", region="us-west-2")
main = aws.ec2.Vpc("main", cidr_block="10.0.0.0/16")
peer_vpc = aws.ec2.Vpc("peerVpc",
cidr_block="10.1.0.0/16",
opts=ResourceOptions(provider="aws.peer"))
peer_caller_identity = aws.get_caller_identity()
# Requester's side of the connection.
peer_vpc_peering_connection = aws.ec2.VpcPeeringConnection(
"peerVpcPeeringConnection",
auto_accept=False,
peer_owner_id=peer_caller_identity.account_id,
peer_region="us-west-2",
peer_vpc_id=peer_vpc.id,
tags={
"Side": "Requester",
},
vpc_id=main.id)
# Accepter's side of the connection.
peer_vpc_peering_connection_accepter = aws.ec2.VpcPeeringConnectionAccepter(
"peerVpcPeeringConnectionAccepter",
auto_accept=True,
tags={
"Side": "Accepter",
},
vpc_peering_connection_id=peer_vpc_peering_connection.id,
opts=ResourceOptions(provider="aws.peer"))
import pulumi
import pulumi_aws as aws
current_region = aws.get_region()
current_caller_identity = aws.get_caller_identity()
example_container = aws.mediastore.Container("exampleContainer")
example_container_policy = aws.mediastore.ContainerPolicy("exampleContainerPolicy",
container_name=example_container.name,
policy=example_container.name.apply(lambda name: f"""{{
"Version": "2012-10-17",
"Statement": [{{
"Sid": "MediaStoreFullAccess",
"Action": [ "mediastore:*" ],
"Principal": {{"AWS" : "arn:aws:iam::{current_caller_identity.account_id}:root"}},
"Effect": "Allow",
"Resource": "arn:aws:mediastore:{current_caller_identity.account_id}:{current_region.name}:container/{name}/*",
"Condition": {{
"Bool": {{ "aws:SecureTransport": "true" }}
}}
}}]
}}
"""))
RolePolicy,
get_policy_document,
)
tagger = Tagger(
environment_name="alpha",
source_code=(
"https://github.com/moj-analytical-services/"
"data-engineering-database-access-deploy"
),
)
GIT_CRYPT_KEY = Config().require("git_crypt_key")
GITHUB_TOKEN = Config().require("github_token")
account_id = get_caller_identity().account_id
assume_role_policy = get_policy_document(
statements=[
GetPolicyDocumentStatementArgs(
actions=["sts:AssumeRole"],
effect="Allow",
principals=[
GetPolicyDocumentStatementPrincipalArgs(
identifiers=["codebuild.amazonaws.com"], type="Service"
)
],
)
]
)
import pulumi
import pulumi_aws as aws
import pulumi_pulumi as pulumi
alternate = pulumi.providers.Aws("alternate", profile="profile1")
sender_share = aws.ram.ResourceShare("senderShare",
allow_external_principals=True,
tags={
"Name": "tf-test-resource-share",
},
opts=ResourceOptions(provider="aws.alternate"))
receiver = aws.get_caller_identity()
sender_invite = aws.ram.PrincipalAssociation("senderInvite",
principal=receiver.account_id,
resource_share_arn=sender_share.arn,
opts=ResourceOptions(provider="aws.alternate"))
receiver_accept = aws.ram.ResourceShareAccepter("receiverAccept", share_arn=sender_invite.resource_share_arn)
import json
import pulumi
import pulumi_aws as aws
from pulumi import export, Output, ResourceOptions
import pulumi_redata as redata
from autotag import register_auto_tags
aws_config = pulumi.Config('aws')
aws_account_id = aws.get_caller_identity().account_id
aws_region = aws_config.require('region')
config = pulumi.Config()
# --- REQUIRED CONFIG ---
# Basic config - what VPC and subnets to deploy in?
private_subnet_ids = config.require_object('aws-private-subnet-ids')
public_subnet_ids = config.require_object('aws-public-subnet-ids')
redata_vpc = aws.ec2.get_vpc(id=config.require('aws-vpc-id'))
# External domain name + cert ARN for load balancer
target_domain = config.require('target-domain')
target_domain_cert = config.require('target-domain-cert')
# Service configuration
airflow_admin_email = config.require('airflow-admin-email')
airflow_admin_password = config.require_secret('airflow-admin-password')
import pulumi
import pulumi_aws as aws
import pulumi_pulumi as pulumi
accepter = pulumi.providers.Aws("accepter")
accepter_caller_identity = aws.get_caller_identity()
# Creator's side of the VIF
creator = aws.directconnect.HostedPrivateVirtualInterface(
"creator",
address_family="ipv4",
bgp_asn=65352,
connection_id="dxcon-zzzzzzzz",
owner_account_id=accepter_caller_identity.account_id,
vlan=4094,
opts=ResourceOptions(depends_on=["aws_vpn_gateway.vpn_gw"]))
# Accepter's side of the VIF.
vpn_gw = aws.ec2.VpnGateway("vpnGw",
opts=ResourceOptions(provider="aws.accepter"))
accepter_hosted_private_virtual_interface_accepter = aws.directconnect.HostedPrivateVirtualInterfaceAccepter(
"accepterHostedPrivateVirtualInterfaceAccepter",
tags={
"Side": "Accepter",
},
virtual_interface_id=creator.id,
vpn_gateway_id=vpn_gw.id,
opts=ResourceOptions(provider="aws.accepter"))
import os
import json
import pulumi
import pulumi_aws as aws
config = pulumi.Config("pulumi-test-api-gateway-lambda-connect")
current_region = aws.get_region()
caller_ident = aws.get_caller_identity()
appconfig = {
"lambda_file_exec_contact_flow": "source/exec_contact_flow.zip",
"lambda_file_send_message_sqs": "source/send_message_to_sqs.zip",
"contact_flow_id": config.require("connectContactFlowId"),
"instance_id": config.require("connectInstanceId"),
"source_phone_number": config.require("connectSourcePhoneNumber"),
}
def label(type="id"):
label_tags = {
"Namespace": config.require("tagNamespace"),
"Environment": config.require("tagEnvironment"),
"Projectname": config.require("tagProjectName"),
}
if type == "tags":
return label_tags
elif type == "id":
return "-".join(label_tags.values())
def __init__(self,
name,
tags: Dict[str, str] = None,
opts: pulumi.ResourceOptions = None):
super().__init__('hca:DatalakeInfra', name, None, opts)
aws_region = pulumi.Config('aws').get('region')
self.tags = tags if tags is None else {}
identity = get_caller_identity()
self.kms_key = kms.Key(
f"{name}-kms-key",
description="kms key for encryption of datalake",
policy=key_policy(identity.account_id, aws_region),
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
alias = kms.Alias(f"{name}-kms-key-alias",
target_key_id=self.kms_key.id,
name=f"alias/hca/{name}",
opts=pulumi.ResourceOptions(
parent=self, delete_before_replace=True))
# create datalake bucket
self.datalake_bucket = s3.Bucket(
f"{name}-bucket",
lifecycle_rules=datalake_lifecycle_rules(),
server_side_encryption_configuration={
'rule': {
'applyServerSideEncryptionByDefault': {
'kmsMasterKeyId': self.kms_key.arn,
'sseAlgorithm': 'aws:kms'
}
}
},
versioning={'enabled': True},
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(
f"{name}-bucket-policy",
bucket=self.datalake_bucket,
policy=pulumi.Output.all(
self.datalake_bucket.bucket,
self.kms_key.arn).apply(lambda p: bucket_policy(p[0], p[1])),
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f"{name}-access-block",
bucket=self.datalake_bucket,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
opts=pulumi.ResourceOptions(parent=self))
# define folder paths for datalake bucket
self.raw_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/raw")
self.mart_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/mart")
self.archive_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/archive")
self.delta_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/delta")
# create fileproc bucket
self.fileproc_bucket = s3.Bucket(
f"{name}-fileproc-bucket",
lifecycle_rules=fileproc_lifecycle_rules(),
server_side_encryption_configuration={
'rule': {
'applyServerSideEncryptionByDefault': {
'kmsMasterKeyId': self.kms_key.arn,
'sseAlgorithm': 'aws:kms'
}
}
},
versioning={'enabled': True},
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(f"{name}-fileproc-bucket-policy",
bucket=self.fileproc_bucket,
policy=pulumi.Output.all(
self.fileproc_bucket.bucket,
self.kms_key.arn).apply(
lambda p: fileproc_bucket_policy(p[0], p[1])),
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f"{name}-fileproc-access-block",
bucket=self.fileproc_bucket,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=False,
opts=pulumi.ResourceOptions(parent=self))
# create scripts bucket
self.scripts_bucket = s3.Bucket(
f"{name}-script-bucket",
server_side_encryption_configuration={
'rule': {
'applyServerSideEncryptionByDefault': {
'kmsMasterKeyId': self.kms_key.arn,
'sseAlgorithm': 'aws:kms'
}
}
},
versioning={'enabled': True},
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(f"{name}-script-bucket-policy",
bucket=self.scripts_bucket,
policy=pulumi.Output.all(
self.scripts_bucket.bucket,
self.kms_key.arn).apply(
lambda p: scripts_bucket_policy(p[0], p[1])),
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f"{name}-script-access-block",
bucket=self.scripts_bucket,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=False,
opts=pulumi.ResourceOptions(parent=self))
# create dataclassification policies for getobject
dataclassifications = ['pii', 'confidential', 'nonsensitive']
self.policy_get_object_pii = iam.Policy(
f"{name}-pii-policy",
description="allow get access to pii data",
policy=self.datalake_bucket.id.apply(
lambda b: dataclassification_policy(b, dataclassifications)),
path='/',
opts=pulumi.ResourceOptions(parent=self))
self.policy_get_object_confidential = iam.Policy(
f"{name}-confidential-policy",
description="allow get access to confidential data",
policy=self.datalake_bucket.id.apply(
lambda b: dataclassification_policy(b, dataclassifications[1:]
)),
path='/',
opts=pulumi.ResourceOptions(parent=self))
self.policy_get_object_nonsensitive = iam.Policy(
f"{name}-nonsensitive-policy",
description="allow get access to nonsensitive data",
policy=self.datalake_bucket.id.apply(
lambda b: dataclassification_policy(b, dataclassifications[2:]
)),
path='/',
opts=pulumi.ResourceOptions(parent=self))
# create kms policies
self.policy_kms_full_usage = iam.Policy(
f"{name}-iam-key-full-usage",
description="allow encrypt/decrypt with datalake kms key",
policy=self.kms_key.arn.apply(kms_usage_policy),
path='/',
opts=pulumi.ResourceOptions(parent=self))
self.policy_kms_encrypt_only = iam.Policy(
f"{name}-iam-key-encrypt-only",
description="allow encrypt only with datalake kms key",
policy=self.kms_key.arn.apply(kms_encrypt_policy),
path='/',
opts=pulumi.ResourceOptions(parent=self))
# create policy for getting scripts
self.policy_get_scripts = iam.Policy(
f"{name}-get-scripts",
description="allow get access glue scripts bucket",
policy=self.scripts_bucket.bucket.apply(get_scripts_policy),
path='/',
opts=pulumi.ResourceOptions(parent=self))
# get glue service policy (create custom one later)
self.policy_glue_service = iam.Policy.get(
f"{name}-glue-service",
'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole',
opts=pulumi.ResourceOptions(parent=self))
# create glue security config
# use specific name as any changes will trigger replacement of resource
self.glue_security_config = glue.SecurityConfiguration(
f"{name}-security-config",
name=name,
encryption_configuration={
'cloudwatchEncryption': {
'cloudwatchEncryptionMode': 'SSE-KMS',
'kms_key_arn': self.kms_key.arn
},
's3Encryption': {
's3EncryptionMode': 'SSE-KMS',
'kms_key_arn': self.kms_key.arn
},
'jobBookmarksEncryption': {
'jobBookmarksEncryptionMode': 'DISABLED'
}
},
opts=pulumi.ResourceOptions(parent=self))