def allow_s3_bucket_access(s3_bucket, roles, lamda_function_arn):
role_arns = [role.arn for role in roles]
bucket_policy = Output.all(s3_bucket.arn,
role_arns).apply(lambda args: json.dumps({
"Version":
"2012-10-17",
"Id":
"MorgueFileBucketPolicy",
"Statement": [
{
"Sid": "AllowThingsInTheBucket",
"Effect": "Allow",
"Principal": {
"AWS": args[1]
},
"Action": "s3:*",
"Resource": f"{args[0]}/*",
},
{
"Sid": "AllowTheBucket",
"Effect": "Allow",
"Principal": {
"AWS": args[1]
},
"Action": "s3:*",
"Resource": f"{args[0]}",
},
],
}))
s3.BucketPolicy("morgue-file-bucket-policy",
bucket=s3_bucket.id,
policy=bucket_policy)
def allow_s3_bucket_access(s3_bucket):
bucket_policy = Output.all(s3_bucket.arn).apply(lambda args: json.dumps({
"Version":
"2012-10-17",
"Id":
"BeginWorldExchange",
"Statement": [
{
"Sid": "PublicAccess",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "s3:Get*",
"Resource": f"{args[0]}/*",
},
{
"Sid": "BeginWriteAccess",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::851075464416:root"
},
"Action": "s3:Put*",
"Resource": f"{args[0]}/*",
},
],
}))
s3.BucketPolicy("beginworld-exchange-bucket-policy",
bucket=s3_bucket.id,
policy=bucket_policy)
def __init__(self,
name: str,
content_dir: str,
index_document: str,
error_document: str,
opts: pulumi.ResourceOptions = None):
super().__init__('StaticWebSite', name, None, opts)
self.name = name
# Create the S3 bucket
self.s3_bucket = s3.Bucket(name,
website={
'index_document': index_document,
'error_document': error_document
})
bucket_name = self.s3_bucket.id
# Copy website content files to the newly created S3 bucket
for file in os.listdir(content_dir):
filepath = os.path.join(content_dir, file)
mime_type, _ = mimetypes.guess_type(filepath)
s3.BucketObject(file,
bucket=bucket_name,
source=FileAsset(filepath),
content_type=mime_type)
# Set bucket policy to enable read access for all users
s3.BucketPolicy("bucket-policy",
bucket=bucket_name,
policy=bucket_name.apply(public_read_policy_for_bucket))
super().register_outputs({})
def pulumi_program():
# Create a bucket and expose a website index document
site_bucket = s3.Bucket("s3-website-bucket", website=s3.BucketWebsiteArgs(index_document="index.html"))
index_content = """
<html>
<head><title>Hello S3</title><meta charset="UTF-8"></head>
<body>
<p>Hello, world!</p>
<p>Made with ❤️ with <a href="https://pulumi.com">Pulumi</a></p>
</body>
</html>
"""
# Write our index.html into the site bucket
s3.BucketObject("index",
bucket=site_bucket.id, # reference to the s3.Bucket object
content=index_content,
key="index.html", # set the key of the object
content_type="text/html; charset=utf-8") # set the MIME type of the file
# Set the access policy for the bucket so all objects are readable
s3.BucketPolicy("bucket-policy", bucket=site_bucket.id, policy={
"Version": "2012-10-17",
"Statement": {
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
# Policy refers to bucket explicitly
"Resource": [pulumi.Output.concat("arn:aws:s3:::", site_bucket.id, "/*")]
},
})
# Export the website URL
pulumi.export("website_url", site_bucket.website_endpoint)
def create_pulumi_program(content: str):
# Create a bucket and expose a website index document
site_bucket = s3.Bucket(
"s3-website-bucket",
website=s3.BucketWebsiteArgs(index_document="index.html"))
index_content = content
# Write our index.html into the site bucket
s3.BucketObject("index",
bucket=site_bucket.id,
content=index_content,
key="index.html",
content_type="text/html; charset=utf-8")
# Set the access policy for the bucket so all objects are readable
s3.BucketPolicy(
"bucket-policy",
bucket=site_bucket.id,
policy={
"Version": "2012-10-17",
"Statement": {
"Effect":
"Allow",
"Principal":
"*",
"Action": ["s3:GetObject"],
# Policy refers to bucket explicitly
"Resource":
[pulumi.Output.concat("arn:aws:s3:::", site_bucket.id, "/*")]
},
})
# Export the website URL
pulumi.export("website_url", site_bucket.website_endpoint)
def create_static_website(bucket_name, title, body):
# Create a bucket and expose a website index document
site_bucket = s3.Bucket(
bucket_name,
bucket=bucket_name,
website=s3.BucketWebsiteArgs(index_document="index.html"))
index_content = f"""
<html>
<head>
<title>{title}</title>
<meta charset="UTF-8"></head>
<body>{body}</body>
</html>
"""
# Write our index.html into the site bucket
s3.BucketObject(
"index",
bucket=site_bucket.id, # reference to the s3.Bucket object
content=index_content,
key="index.html", # set the key of the object
content_type="text/html; charset=utf-8"
) # set the MIME type of the file
# Set the access policy for the bucket so all objects are readable
s3.BucketPolicy(
"bucket-policy",
bucket=site_bucket.id,
policy={
"Version": "2012-10-17",
"Statement": {
"Effect":
"Allow",
"Principal":
"*",
"Action": ["s3:GetObject"],
# Policy refers to bucket explicitly
"Resource":
[pulumi.Output.concat("arn:aws:s3:::", site_bucket.id, "/*")]
},
})
# Export the website URL
pulumi.export("website_url", site_bucket.website_endpoint)
def _create_bucket(self, name, origin_access_identity):
bucket = s3.Bucket(f'website-{self.name}-{self.stack}-{name}',
acl='private',
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f'website-{self.name}-{self.stack}-{name}',
bucket=bucket.id,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(f'website-{self.name}-{self.stack}-{name}-policy',
bucket=bucket.id,
policy=pulumi.Output.all(
origin_access_identity.iam_arn,
bucket.arn).apply(self._get_s3_policy),
opts=pulumi.ResourceOptions(parent=self))
return bucket
def create_s3website_bucket(bucket_name):
bucket = s3.Bucket(
bucket_name,
bucket=bucket_name,
acl="public-read",
website=s3.BucketWebsiteArgs(
index_document="index.html",
error_document="404.html",
),
tags={
"Website": WEBSITE_DOMAIN_NAME,
"Environment": "dev",
},
)
bucket_id = bucket.id
bucket_policy = s3.BucketPolicy(
bucket_name + "-bucket-policy",
bucket=bucket_id,
policy=bucket_id.apply(public_access_policy_for_s3website_bucket))
return bucket
def __init__(self,
name: str,
args: StaticPageArgs,
props: Optional[dict] = None,
opts: Optional[ResourceOptions] = None) -> None:
super().__init__('xyz:index:StaticPage', name, props, opts)
# Create a bucket and expose a website index document.
bucket = s3.Bucket(
f'{name}-bucket',
website=s3.BucketWebsiteArgs(index_document='index.html'),
opts=ResourceOptions(parent=self))
# Create a bucket object for the index document.
s3.BucketObject(
f'{name}-index-object',
bucket=bucket.bucket,
key='index.html',
content=args.index_content,
content_type='text/html',
opts=ResourceOptions(parent=bucket))
# Set the access policy for the bucket so all objects are readable.
s3.BucketPolicy(
f'{name}-bucket-policy',
bucket=bucket.bucket,
policy=bucket.bucket.apply(_allow_getobject_policy),
opts=ResourceOptions(parent=bucket))
self.bucket = bucket
self.website_url = bucket.website_endpoint
self.register_outputs({
'bucket': bucket,
'websiteUrl': bucket.website_endpoint,
})
"Action": "s3:PutObject",
"Resource":
f"arn:aws:s3:::{bucket_name}/prefix/AWSLogs/{account_id}/*",
"Condition": {
"StringEquals": {
"s3:x-amz-acl": "bucket-owner-full-control"
}
}
}]
})
# apply policy to bucket
bucket_name = bucket.id
bucket_policy = s3.BucketPolicy(
"bucket-policy",
bucket=bucket_name,
policy=bucket_name.apply(bucket_policy_cloudtrial))
# s3 Block Public access
s3Access = s3.BucketPublicAccessBlock("Block Public access",
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
bucket=bucket.id)
# Create cloudwatch log
cloudwatch_log = cloudwatch.LogGroup('LogGroup', name='S3BucketActivity')
# add log stream to cloudwatch log
log_stream = cloudwatch.LogStream("CloudWatch_log_stream",
obj = s3.BucketObject(file,
bucket=web_bucket.id,
source=FileAsset(filepath),
content_type=mime_type)
def public_read_policy_for_bucket(bucket_name):
return json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource": [
f"arn:aws:s3:::{bucket_name}/*",
]
}]
})
bucket_name = web_bucket.id
bucket_policy = s3.BucketPolicy(
"bucket-policy",
bucket=bucket_name,
policy=bucket_name.apply(public_read_policy_for_bucket))
# Export the name of the bucket
export('bucket_name', web_bucket.id)
export('website_url', web_bucket.website_endpoint)
"my-website-bucket",
website=s3.BucketWebsiteArgs(index_document="index.html", ),
)
content_dir = "www"
for file in os.listdir(content_dir):
filepath = os.path.join(content_dir, file)
mime_type, _ = mimetypes.guess_type(filepath)
obj = s3.BucketObject(file,
bucket=bucket.id,
source=pulumi.FileAsset(filepath),
content_type=mime_type,
opts=pulumi.ResourceOptions(parent=bucket))
bucket_policy = s3.BucketPolicy(
"my-website-bucket-policy",
bucket=bucket.id,
policy=bucket.arn.apply(lambda arn: json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource": [f"{arn}/*"]
}]
})),
opts=pulumi.ResourceOptions(parent=bucket))
pulumi.export('website_url', bucket.website_endpoint)
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": [
"s3:GetObject"
],
"Resource": [
f"arn:aws:s3:::{bucket_name}/*",
]
}]
})
web_bucket_id = web_bucket.id
web_bucket_policy = s3.BucketPolicy("bucket-policy",
bucket=web_bucket_id,
policy=web_bucket_id.apply(public_read_policy_for_bucket))
# Split a domain name into its subdomain and parent domain names.
# e.g. "www.example.com" => "www", "example.com".
def get_domain_and_subdomain(domain):
names = domain.split(".")
if len(names) < 3:
return ('', domain)
subdomain = names[0]
parent_domain = ".".join(names[1:])
return (subdomain, parent_domain)
(subdomain, parent_domain) = get_domain_and_subdomain(domain_name)
"Resource":
f"arn:aws:s3:::{bucket_name}/prefix/AWSLogs/{account_id}/*",
"Condition": {
"StringEquals": {
"s3:x-amz-acl": "bucket-owner-full-control"
}
}
}]
})
# apply policy to bucket
bucket_name = bucket.id
bucket_policy = s3.BucketPolicy(
"bucket-policy",
opts=ResourceOptions(depends_on=[bucket]),
bucket=bucket_name,
policy=bucket_name.apply(bucket_policy_cloudtrial))
# s3 Block Public access
s3Access = s3.BucketPublicAccessBlock(
"Block Public access",
opts=ResourceOptions(depends_on=[bucket_policy]),
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
bucket=bucket.id)
# Create cloudwatch log
cloudwatch_log = cloudwatch.LogGroup('LogGroup', name='S3BucketActivity')
def StaticSite(self, name, domain, zone, content_dir, __opts__):
"""
A static site, at the given domain with the contents of the given directory.
Uses S3, CloudFront, ACM, and Route53.
"""
web_bucket = s3.Bucket(
f'{name}-bucket',
bucket=domain,
website={
"index_document": "index.html",
"errorDocument": "404.html",
},
acl='public-read',
website_domain=domain,
**opts(parent=self),
)
for absp, relp in walk(content_dir):
mime_type, _ = mimetypes.guess_type(str(absp))
s3.BucketObject(
f'{name}-{relp!s}',
key=str(relp),
bucket=web_bucket.id,
source=FileAsset(str(absp)),
content_type=mime_type,
**opts(parent=web_bucket),
)
bucket_name = web_bucket.id
s3.BucketPolicy(
f"{name}-policy",
bucket=bucket_name,
policy=bucket_name.apply(public_read_policy_for_bucket),
**opts(parent=web_bucket),
)
cert = Certificate(
f"{name}-cert",
domain=domain,
zone=zone,
**opts(parent=self, region='us-east-1'),
)
distro = cloudfront.Distribution(
f"{name}-dist",
enabled=True,
# Alternate aliases the CloudFront distribution can be reached at, in addition to https://xxxx.cloudfront.net.
# Required if you want to access the distribution via config.targetDomain as well.
aliases=[domain],
is_ipv6_enabled=True,
# We only specify one origin for this distribution, the S3 content bucket.
origins=[
{
"originId": web_bucket.arn,
"domainName": web_bucket.website_endpoint,
"customOriginConfig": {
# Amazon S3 doesn't support HTTPS connections when using an S3 bucket configured as a website endpoint.
"originProtocolPolicy": "http-only",
"httpPort": 80,
"httpsPort": 443,
"originSslProtocols": ["TLSv1.2"],
},
},
],
default_root_object="index.html",
# A CloudFront distribution can configure different cache behaviors based on the request path.
# Here we just specify a single, default cache behavior which is just read-only requests to S3.
default_cache_behavior={
"targetOriginId": web_bucket.arn,
"viewerProtocolPolicy": "redirect-to-https",
"allowedMethods": ["GET", "HEAD", "OPTIONS"],
"cachedMethods": ["GET", "HEAD", "OPTIONS"],
"forwardedValues": {
"cookies": {"forward": "none"},
"queryString": False,
},
"minTtl": 0,
"defaultTtl": 10*60,
"maxTtl": 10*60,
},
# "All" is the most broad distribution, and also the most expensive.
# "100" is the least broad, and also the least expensive.
price_class="PriceClass_100",
# You can customize error responses. When CloudFront recieves an error from the origin (e.g.
# S3 or some other web service) it can return a different error code, and return the
# response for a different resource.
custom_error_responses=[
{"errorCode": 404, "responseCode": 404, "responsePagePath": "/404.html"},
],
restrictions={
"geoRestriction": {
"restrictionType": "none",
},
},
viewer_certificate={
"acmCertificateArn": cert.cert_arn,
"sslSupportMethod": "sni-only",
},
# loggingConfig: {
# bucket: logsBucket.bucketDomainName,
# includeCookies: false,
# prefix: `${config.targetDomain}/`,
# },
**opts(parent=self),
)
a_aaaa(
f"{name}-record",
name=domain,
zone_id=zone.zone_id,
aliases=[
{
'name': distro.domain_name,
'zone_id': distro.hosted_zone_id,
'evaluate_target_health': True,
},
],
**opts(parent=self),
)
return {
'url': f'https://{domain}/'
}
def __init__(self,
name,
tags: Dict[str, str] = None,
opts: pulumi.ResourceOptions = None):
super().__init__('hca:DatalakeInfra', name, None, opts)
aws_region = pulumi.Config('aws').get('region')
self.tags = tags if tags is None else {}
identity = get_caller_identity()
self.kms_key = kms.Key(
f"{name}-kms-key",
description="kms key for encryption of datalake",
policy=key_policy(identity.account_id, aws_region),
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
alias = kms.Alias(f"{name}-kms-key-alias",
target_key_id=self.kms_key.id,
name=f"alias/hca/{name}",
opts=pulumi.ResourceOptions(
parent=self, delete_before_replace=True))
# create datalake bucket
self.datalake_bucket = s3.Bucket(
f"{name}-bucket",
lifecycle_rules=datalake_lifecycle_rules(),
server_side_encryption_configuration={
'rule': {
'applyServerSideEncryptionByDefault': {
'kmsMasterKeyId': self.kms_key.arn,
'sseAlgorithm': 'aws:kms'
}
}
},
versioning={'enabled': True},
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(
f"{name}-bucket-policy",
bucket=self.datalake_bucket,
policy=pulumi.Output.all(
self.datalake_bucket.bucket,
self.kms_key.arn).apply(lambda p: bucket_policy(p[0], p[1])),
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f"{name}-access-block",
bucket=self.datalake_bucket,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=True,
opts=pulumi.ResourceOptions(parent=self))
# define folder paths for datalake bucket
self.raw_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/raw")
self.mart_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/mart")
self.archive_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/archive")
self.delta_location = self.datalake_bucket.bucket.apply(
lambda b: f"s3://{b}/delta")
# create fileproc bucket
self.fileproc_bucket = s3.Bucket(
f"{name}-fileproc-bucket",
lifecycle_rules=fileproc_lifecycle_rules(),
server_side_encryption_configuration={
'rule': {
'applyServerSideEncryptionByDefault': {
'kmsMasterKeyId': self.kms_key.arn,
'sseAlgorithm': 'aws:kms'
}
}
},
versioning={'enabled': True},
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(f"{name}-fileproc-bucket-policy",
bucket=self.fileproc_bucket,
policy=pulumi.Output.all(
self.fileproc_bucket.bucket,
self.kms_key.arn).apply(
lambda p: fileproc_bucket_policy(p[0], p[1])),
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f"{name}-fileproc-access-block",
bucket=self.fileproc_bucket,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=False,
opts=pulumi.ResourceOptions(parent=self))
# create scripts bucket
self.scripts_bucket = s3.Bucket(
f"{name}-script-bucket",
server_side_encryption_configuration={
'rule': {
'applyServerSideEncryptionByDefault': {
'kmsMasterKeyId': self.kms_key.arn,
'sseAlgorithm': 'aws:kms'
}
}
},
versioning={'enabled': True},
tags=self.tags,
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPolicy(f"{name}-script-bucket-policy",
bucket=self.scripts_bucket,
policy=pulumi.Output.all(
self.scripts_bucket.bucket,
self.kms_key.arn).apply(
lambda p: scripts_bucket_policy(p[0], p[1])),
opts=pulumi.ResourceOptions(parent=self))
s3.BucketPublicAccessBlock(f"{name}-script-access-block",
bucket=self.scripts_bucket,
block_public_acls=True,
block_public_policy=True,
ignore_public_acls=True,
restrict_public_buckets=False,
opts=pulumi.ResourceOptions(parent=self))
# create dataclassification policies for getobject
dataclassifications = ['pii', 'confidential', 'nonsensitive']
self.policy_get_object_pii = iam.Policy(
f"{name}-pii-policy",
description="allow get access to pii data",
policy=self.datalake_bucket.id.apply(
lambda b: dataclassification_policy(b, dataclassifications)),
path='/',
opts=pulumi.ResourceOptions(parent=self))
self.policy_get_object_confidential = iam.Policy(
f"{name}-confidential-policy",
description="allow get access to confidential data",
policy=self.datalake_bucket.id.apply(
lambda b: dataclassification_policy(b, dataclassifications[1:]
)),
path='/',
opts=pulumi.ResourceOptions(parent=self))
self.policy_get_object_nonsensitive = iam.Policy(
f"{name}-nonsensitive-policy",
description="allow get access to nonsensitive data",
policy=self.datalake_bucket.id.apply(
lambda b: dataclassification_policy(b, dataclassifications[2:]
)),
path='/',
opts=pulumi.ResourceOptions(parent=self))
# create kms policies
self.policy_kms_full_usage = iam.Policy(
f"{name}-iam-key-full-usage",
description="allow encrypt/decrypt with datalake kms key",
policy=self.kms_key.arn.apply(kms_usage_policy),
path='/',
opts=pulumi.ResourceOptions(parent=self))
self.policy_kms_encrypt_only = iam.Policy(
f"{name}-iam-key-encrypt-only",
description="allow encrypt only with datalake kms key",
policy=self.kms_key.arn.apply(kms_encrypt_policy),
path='/',
opts=pulumi.ResourceOptions(parent=self))
# create policy for getting scripts
self.policy_get_scripts = iam.Policy(
f"{name}-get-scripts",
description="allow get access glue scripts bucket",
policy=self.scripts_bucket.bucket.apply(get_scripts_policy),
path='/',
opts=pulumi.ResourceOptions(parent=self))
# get glue service policy (create custom one later)
self.policy_glue_service = iam.Policy.get(
f"{name}-glue-service",
'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole',
opts=pulumi.ResourceOptions(parent=self))
# create glue security config
# use specific name as any changes will trigger replacement of resource
self.glue_security_config = glue.SecurityConfiguration(
f"{name}-security-config",
name=name,
encryption_configuration={
'cloudwatchEncryption': {
'cloudwatchEncryptionMode': 'SSE-KMS',
'kms_key_arn': self.kms_key.arn
},
's3Encryption': {
's3EncryptionMode': 'SSE-KMS',
'kms_key_arn': self.kms_key.arn
},
'jobBookmarksEncryption': {
'jobBookmarksEncryptionMode': 'DISABLED'
}
},
opts=pulumi.ResourceOptions(parent=self))
