def _lambda_function_from_asset(self,
lambda_asset,
function_name,
function_cmd,
environment,
*,
reserved_concurrent_executions=2,
timeout_minutes=15,
memory_size=128,
max_event_age_minutes=60):
return _lambda.DockerImageFunction(
self,
function_name,
function_name=function_name,
environment=environment,
log_retention=logs.RetentionDays.ONE_WEEK,
memory_size=memory_size,
reserved_concurrent_executions=reserved_concurrent_executions,
timeout=cdk.Duration.minutes(timeout_minutes),
tracing=_lambda.Tracing.ACTIVE,
max_event_age=cdk.Duration.minutes(max_event_age_minutes),
code=_lambda.DockerImageCode.from_ecr(
lambda_asset.repository,
cmd=[function_cmd],
tag=lambda_asset.image_uri.split(':')[1]))
def __init__(self,
scope: core.Construct,
id: str,
repository_name: str,
directory: str,
subnet_group_name: str,
context: InfraContext,
securityGroups: typing.Optional[typing.List[
ec2.SecurityGroup]] = None,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.repo = assets.DockerImageAsset(self,
'Repo',
directory=os.path.join(
src_root_dir, directory),
repository_name=repository_name)
self.function = lambda_.DockerImageFunction(
self,
'ContainerFunction',
code=lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1]
), # lambda_.DockerImageCode.from_image_asset(directory=os.path.join(src_root_dir,directory)),
description='Python container lambda function for ' +
repository_name,
timeout=core.Duration.minutes(1),
tracing=lambda_.Tracing.ACTIVE,
vpc=context.networking.vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_group_name=subnet_group_name),
security_groups=securityGroups)
def __init__(self, scope: core.Construct, construct_id: str,
repo_name: str, repo_tag: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# 1. Roles & Policies
role = iam.Role(
self,
'myappRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'))
role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["*"],
actions=['events:*']))
role.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=["arn:aws:iam::*:role/AWS_Events_Invoke_Targets"],
actions=['iam:PassRole']))
role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["*"],
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream", "logs:PutLogEvents"
]))
role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["*"],
actions=["s3:*"]))
role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["*"],
actions=["lambda:*"]))
role.add_to_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["*"],
actions=["sns:*"]))
# 2. Lambda functions
repo = ecr.Repository.from_repository_name(
self, "Repository", repository_name=f"{repo_name}")
lambdaFn1 = lambdas.DockerImageFunction(
self,
"lambda_scraper_cdk_func_1",
code=lambdas.DockerImageCode.from_ecr(repository=repo,
tag=repo_tag),
timeout=core.Duration.seconds(600),
memory_size=2048,
environment=dict(PATH="/opt"),
role=role)
def __init__(self, app: core.App, _id: str):
super().__init__(scope=app, id=_id)
_ = lambda_.DockerImageFunction(
scope=self,
id="container_function",
code=lambda_.DockerImageCode.from_image_asset(
directory="docker",
repository_name="lambda_container_example"))
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
fnc = lambda_.DockerImageFunction(
self,
"AssetFunction",
code=lambda_.DockerImageCode.from_image_asset(
os.path.join(dirname, "docker-handler")
),
)
def __init__(self,
scope: core.Construct,
id: str,
resources: FsiSharedResources,
subnet_group_name: str = 'Default',
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Configure role...
role = iam.Role(self,
'Role',
assumed_by=iam.ServicePrincipal(service='lambda'),
description='Ameritrade Secrets Rotation Lambda via ' +
self.component_name,
role_name='{}@homenet-{}.{}'.format(
self.component_name,
resources.landing_zone.zone_name,
core.Stack.of(self).region),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name=
'service-role/AWSLambdaVPCAccessExecutionRole')
])
resources.tda_secret.grant_write(role)
# Configure the lambda...
self.repo = assets.DockerImageAsset(
self, 'Repo', directory='src/fsi/secret-rotation')
code = lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1])
self.function = lambda_.DockerImageFunction(
self,
'Function',
code=code,
role=role,
function_name='HomeNet-Fsi{}-{}'.format(
resources.landing_zone.zone_name, self.component_name),
description='Python container function for ' + self.component_name,
timeout=core.Duration.minutes(15),
tracing=lambda_.Tracing.ACTIVE,
vpc=resources.landing_zone.vpc,
log_retention=logs.RetentionDays.TWO_WEEKS,
memory_size=3 * 128,
allow_all_outbound=True,
vpc_subnets=ec2.SubnetSelection(
subnet_group_name=subnet_group_name),
security_groups=[resources.landing_zone.security_group],
environment={})
resources.tda_secret.grant_write(self.function.role)
resources.tda_secret.grant_read(self.function.role)
def __init__(self, scope: core.Construct, id: str,
infra: RtspBaseResourcesConstruct, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.infra = infra
self.repo = assets.DockerImageAsset(self,
'Repo',
directory=self.source_directory,
file='Dockerfile')
code = lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1])
role = iam.Role(self,
'Role',
assumed_by=iam.ServicePrincipal(service='lambda'),
description='RtspFunction for the ' +
self.component_name + ' component.',
role_name='{}@homenet-{}.{}'.format(
self.component_name, infra.landing_zone.zone_name,
core.Stack.of(self).region),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name=
'service-role/AWSLambdaVPCAccessExecutionRole')
])
self.function = lambda_.DockerImageFunction(
self,
'Function',
code=code,
role=role,
function_name='HomeNet-{}-{}'.format(infra.landing_zone.zone_name,
self.component_name),
description='Python container lambda function for ' +
self.component_name,
timeout=self.function_timeout,
tracing=lambda_.Tracing.ACTIVE,
vpc=infra.landing_zone.vpc,
log_retention=RetentionDays.FIVE_DAYS,
memory_size=128,
allow_all_outbound=True,
vpc_subnets=ec2.SubnetSelection(
subnet_group_name=infra.subnet_group_name),
security_groups=[infra.security_group],
environment={
'REGION': core.Stack.of(self).region,
})
self.dlq = sqs.Queue(self,
'DeadLetterQueue',
queue_name=self.function.function_name + "_dlq")
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# defines an AWS Lambda resource
model_folder = os.path.dirname(os.path.realpath(__file__)) + "/../model"
predictive_lambda = _lambda.DockerImageFunction(self, 'PredictiveLambda',
code=_lambda.DockerImageCode.from_image_asset(model_folder),
memory_size=4096,
timeout=core.Duration.seconds(15))
# defines an API Gateway Http API resource backed by our "PredictiveLambda" function.
api = api_gw.HttpApi(self, 'PredictiveLambdaEndpoint',
default_integration=integrations.LambdaProxyIntegration(handler=predictive_lambda));
core.CfnOutput(self, 'HTTP API Url', value=api.url);
def __init__(self, scope: cdk.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
lambda_func = lambda_.DockerImageFunction(
scope=self,
id="tso-lambda-function",
code=lambda_.DockerImageCode.from_image_asset(
directory="../docker"),
memory_size=3000,
timeout=core.Duration.seconds(60))
api = apigateway.LambdaRestApi(self,
"tso-apigateway",
handler=lambda_func)
def __init__(self,
scope,
id,
*,
name=None,
directory=None,
bucket=None,
key=None) -> None:
super().__init__(scope, id)
# ==================================================
# ================= IAM ROLE =======================
# ==================================================
lambda_role = iam.Role(
scope=self,
id='lambda_role',
assumed_by=iam.ServicePrincipal(service='lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSLambdaExecute')
])
# ==================================================
# =================== ECR IMAGE ====================
# ==================================================
ecr_image = aws_lambda.DockerImageCode.from_image_asset(
repository_name=name, directory=directory)
# ==================================================
# ================ LAMBDA FUNCTION =================
# ==================================================
self.lambda_function = aws_lambda.DockerImageFunction(
scope=self,
id='lambda',
function_name=name,
code=ecr_image,
memory_size=1024,
role=lambda_role,
environment={
'BUCKET': bucket,
'KEY': key
},
timeout=core.Duration.seconds(60))
def __init__(self,scope:core.Construct, id:str, datalake:DataLakeLayer, project_name:str, concurrency:int=5, **kwargs) ->None:
super().__init__(scope,id,**kwargs)
self.__datalake = datalake
repo = ecr.DockerImageAsset(self,'Repo',
directory=os.path.join(root_dir, project_name),
repository_name=project_name)
self.function = lambda_.DockerImageFunction(self,project_name+'-repo',
code = lambda_.DockerImageCode.from_ecr(
repository=repo.repository,
tag=repo.image_uri.split(':')[-1]), # lambda_.DockerImageCode.from_image_asset(directory=os.path.join(src_root_dir,directory)),
description='Python container lambda function for '+repo.repository.repository_name,
timeout= core.Duration.minutes(15),
memory_size=4096,
tracing= lambda_.Tracing.ACTIVE,
# Note: This throttles the AWS S3 batch job.
# Downloading too fast will cause f-droid to disconnect the crawler
reserved_concurrent_executions= concurrency,
filesystem= lambda_.FileSystem.from_efs_access_point(
ap= self.datalake.efs.add_access_point(
project_name,
path='/'+project_name,
create_acl=efs.Acl(owner_gid="0", owner_uid="0", permissions="777")),
mount_path='/mnt/efs'
),
environment={
'EFS_MOUNT':'/mnt/efs'
},
vpc= self.datalake.vpc)
for name in [
'AmazonElasticFileSystemClientFullAccess',
'AWSXrayWriteOnlyAccess',
'AmazonS3FullAccess',
'AWSCodeCommitFullAccess',
'AmazonCodeGuruReviewerFullAccess' ]:
self.function.role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(name))
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
my_api_function = _lambda.DockerImageFunction(
self,
'MyApiFunction',
code=_lambda.DockerImageCode.from_image_asset('../src'),
timeout=core.Duration.seconds(30),
memory_size=2048)
my_default_integration = apigv2int.LambdaProxyIntegration(
handler=my_api_function)
my_http_api = apigv2.HttpApi(
self, 'MyApi', default_integration=my_default_integration)
core.CfnOutput(self, 'MyApiUrl', value=my_http_api.api_endpoint)
core.CfnOutput(self,
'MyApiFnLogGroup',
value=my_api_function.log_group.log_group_name)
def __init__(self, scope: core.Construct, id: str, *, docker_root: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
base_lambda = _lambda.DockerImageFunction(
self,
"FastAPIImageLambda",
code=_lambda.DockerImageCode.from_image_asset(docker_root),
)
base_api = _apigw.HttpApi(
self,
"FastAPIProxyGateway",
api_name="FastAPIProxyGateway",
default_integration=_apigw_integration.LambdaProxyIntegration(
handler=base_lambda),
)
core.CfnOutput(self,
"EndpointUrl",
value=base_api.api_endpoint,
export_name="fastApiUrl")
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
bucket = _s3.Bucket(self, id='hello-s3-bucket')
hello_lambda = _lambda.DockerImageFunction(
self,
id='hello-s3-lambda',
code=_lambda.DockerImageCode.from_image_asset(
path.realpath('./hello_s3_lambda')),
environment={'OUTPUT_BUCKET_NAME': bucket.bucket_name})
sched_event = _events.Rule(self,
id='hello-s3-sched',
schedule=_events.Schedule.rate(
core.Duration.minutes(5)))
sched_event.add_target(_targets.LambdaFunction(hello_lambda))
# Permissions
bucket.grant_write(hello_lambda)
def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# Policies
s3_access_policy = iam.ManagedPolicy.from_managed_policy_arn(
self,
id="s3_access_policy",
managed_policy_arn="arn:aws:iam::aws:policy/AmazonS3FullAccess")
lambda_access_policy = iam.ManagedPolicy.from_managed_policy_arn(
self,
id="lambda_access_policy",
managed_policy_arn="arn:aws:iam::aws:policy/AWSLambda_FullAccess")
logs_policy = iam.ManagedPolicy.from_managed_policy_arn(
self,
id="logs_policy",
managed_policy_arn=
"arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")
# Roles
lambda_role = iam.Role(
self,
id="lambda_role",
assumed_by=iam.ServicePrincipal(service="lambda.amazonaws.com"),
managed_policies=[
s3_access_policy, lambda_access_policy, logs_policy
],
role_name=f"midi-to-mp3-lambda-role")
# SQS
conversion_sqs = sqs.Queue(self,
id=f"conversion_sqs",
queue_name=f"conversion_sqs",
visibility_timeout=core.Duration.hours(12),
retention_period=core.Duration.days(1))
kickoff_sqs = sqs.Queue(self,
id=f"kickoff_sqs",
queue_name=f"kickoff_sqs",
visibility_timeout=core.Duration.hours(12),
retention_period=core.Duration.days(1))
# S3
midi_file_dropoff_bucket = s3.Bucket(
self,
id="midi_files_dropoff",
bucket_name=MIDI_FILE_DROPOFF_BUCKET,
auto_delete_objects=True,
removal_policy=core.RemovalPolicy.DESTROY)
midi_file_dropoff_bucket.add_event_notification(
event=s3.EventType.OBJECT_CREATED,
dest=s3n.SqsDestination(kickoff_sqs))
created_mp3_files_bucket = s3.Bucket(
self,
id="created_mp3_files",
bucket_name=CREATED_MP3_FILES_BUCKET,
auto_delete_objects=True,
removal_policy=core.RemovalPolicy.DESTROY)
component_midi_files_bucket = s3.Bucket(
self,
id="component_midi_files",
bucket_name=COMPONENT_MIDI_FILES_BUCKET,
auto_delete_objects=True,
removal_policy=core.RemovalPolicy.DESTROY)
component_midi_files_bucket.add_event_notification(
event=s3.EventType.OBJECT_CREATED,
dest=s3n.SqsDestination(conversion_sqs))
# Lambdas
lambda_code = lambda_.DockerImageCode.from_image_asset(
directory='./midi_to_mp3_lambda/',
file="Dockerfile",
build_args={
"AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID"),
"AWS_SECRET_ACCESS_KEY":
os.environ.get("AWS_SECRET_ACCESS_KEY")
})
midi_to_mp3_lambda = lambda_.DockerImageFunction(
self,
id="midi_to_mp3_lambda",
role=lambda_role,
function_name="midi-to-mp3",
memory_size=1024,
timeout=core.Duration.minutes(5),
code=lambda_code)
midi_split_lambda = alg.GoFunction(
self,
id="midi_split_lambda",
entry="./midi_split_lambda/midi_split_lambda.go",
timeout=core.Duration.minutes(15),
runtime=lambda_.Runtime.GO_1_X,
role=lambda_role,
function_name=f"midi-split-lambda",
memory_size=512,
bundling={"environment": {
"GO111MODULE": "off"
}})
s3_cleanup_lambda = lambda_.Function(
self,
id="s3_cleanup_lambda",
runtime=lambda_.Runtime.PYTHON_3_8,
role=lambda_role,
function_name="s3-cleanup-lambda",
memory_size=256,
timeout=core.Duration.minutes(5),
environment={"NUM_WEEKS_TO_KEEP_FILES": "1"},
handler="s3_cleanup_lambda.handler",
code=lambda_.Code.from_asset(os.path.join(".",
"s3_cleanup_lambda")))
weekly_on_sunday_cron = events.Rule(
self,
"Rule",
schedule=events.Schedule.cron(minute='0',
hour='0',
week_day="SUN",
month='*',
year='*'),
)
weekly_on_sunday_cron.add_target(
targets.LambdaFunction(s3_cleanup_lambda))
# Event Sources
midi_to_mp3_lambda.add_event_source(
eventsources.SqsEventSource(queue=conversion_sqs))
midi_split_lambda.add_event_source(
eventsources.SqsEventSource(queue=kickoff_sqs))
def __init__(self, scope: core.Construct, id: str, resources:FsiSharedResources, subnet_group_name:str='Default', **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Configure the container resources...
self.repo = assets.DockerImageAsset(self,'Repo',
directory='src/fsi/account-linking',
file='Dockerfile')
code = lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1])
# Configure security policies...
role = iam.Role(self,'Role',
assumed_by=iam.ServicePrincipal(service='lambda'),
description='HomeNet-{}-Fsi-AccountLinking'.format(resources.landing_zone.zone_name),
role_name='fsi-accountlinking@homenet.{}.{}'.format(
resources.landing_zone.zone_name,
core.Stack.of(self).region),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name='service-role/AWSLambdaVPCAccessExecutionRole'),
])
# Grant any permissions...
resources.tda_secret.grant_write(role)
# Define any variables for the function
self.function_env = {
'REGION': core.Stack.of(self).region,
'TDA_SECRET_ID': resources.tda_secret.secret_arn,
'TDA_REDIRECT_URI': ssm.StringParameter.from_string_parameter_name(self,'TDA_REDIRECT_URI',
string_parameter_name='/HomeNet/Amertitrade/redirect_uri').string_value,
'TDA_CLIENT_ID': ssm.StringParameter.from_string_parameter_name(self, 'TDA_CLIENT_ID',
string_parameter_name='/HomeNet/Ameritrade/client_id').string_value
}
# Create the backing webapi compute ...
self.function = lambda_.DockerImageFunction(self,'Function',
code = code,
role= role,
function_name='HomeNet-{}-Fsi-{}'.format(
resources.landing_zone.zone_name,
FsiAmeritradeAuthGateway.__name__),
description='Python Lambda function for '+FsiAmeritradeAuthGateway.__name__,
timeout= core.Duration.seconds(30),
tracing= lambda_.Tracing.ACTIVE,
vpc= resources.landing_zone.vpc,
log_retention= logs.RetentionDays.FIVE_DAYS,
memory_size=128,
allow_all_outbound=True,
vpc_subnets=ec2.SubnetSelection(subnet_group_name=subnet_group_name),
security_groups=[resources.landing_zone.security_group],
environment=self.function_env,
)
# Bind APIG to Lambda compute...
self.frontend_proxy = a.LambdaRestApi(self,'ApiGateway',
proxy=True,
handler=self.function,
options=a.RestApiProps(
description='Hosts the Ameritrade Auth Callback via '+self.function.function_name,
domain_name= a.DomainNameOptions(
domain_name='auth.trader.fsi',
certificate=Certificate.from_certificate_arn(self,'Certificate',
certificate_arn= 'arn:aws:acm:us-east-2:581361757134:certificate/0d1fc756-ebd6-4660-83a8-814c0976a8c2'),
security_policy= a.SecurityPolicy.TLS_1_0),
policy= iam.PolicyDocument(
statements=[
iam.PolicyStatement(
effect= iam.Effect.ALLOW,
actions=['execute-api:Invoke'],
principals=[iam.AnyPrincipal()],
resources=['*'],
conditions={
'IpAddress':{
'aws:SourceIp': ['10.0.0.0/8','192.168.0.0/16','72.90.160.65/32']
}
}
)
]
),
endpoint_configuration= a.EndpointConfiguration(
types = [ a.EndpointType.REGIONAL],
)
))
# Register Dns Name
r53.ARecord(self,'AliasRecord',
zone=resources.trader_dns_zone,
record_name='auth.%s' % resources.trader_dns_zone.zone_name,
target= r53.RecordTarget.from_alias(dns_targets.ApiGateway(self.frontend_proxy)))
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
self.ssm_signing_secret = ssm.StringParameter(self,
"SLACK_SIGNING_SECRET",
string_value="xxxx")
self.ssm_bot_token = ssm.StringParameter(self,
"SLACK_BOT_TOKEN",
string_value="xoxb-xxxx")
self.ssm_kibela_team = ssm.StringParameter(self,
"KIBELA_TEAM",
string_value="teamname")
self.ssm_kibela_token = ssm.StringParameter(self,
"KIBELA_TOKEN",
string_value="secret/xxxx")
self.public_s3 = s3.Bucket(self,
"PublicS3Bucket",
public_read_access=True)
self.private_s3 = s3.Bucket(self, "PrivateS3Bucket")
self.step_lambda = lambda_.DockerImageFunction(
self,
"StepFunction",
code=lambda_.DockerImageCode.from_image_asset(
path.join(path.dirname(__name__), "../wordcloud-app"),
cmd=["app.handler"],
entrypoint=["/usr/local/bin/python", "-m", "awslambdaric"],
),
environment={
"SSM_KIBELA_TEAM": self.ssm_kibela_team.parameter_name,
"SSM_KIBELA_TOKEN": self.ssm_kibela_token.parameter_name,
"S3_PUBLIC": self.public_s3.bucket_name,
"S3_PRIVATE": self.private_s3.bucket_name,
},
log_retention=logs.RetentionDays.FIVE_DAYS,
timeout=core.Duration.seconds(600),
memory_size=2048,
)
self.create_enumerate_statemachine()
self.create_update_statemachine()
self.create_unfurl_statemachine()
self.bolt_lambda = PythonFunction(
self,
"BoltFunction",
entry=path.join(path.dirname(__name__), "../bolt-app/app"),
index="app.py",
handler="handler",
runtime=lambda_.Runtime.PYTHON_3_8,
environment={
# "SLACK_SIGNING_SECRET": self.secret.secret_value_from_json(
# "SLACK_SIGNING_SECRET"
# ).to_string(),
# "SLACK_BOT_TOKEN": self.secret.secret_value_from_json(
# "SLACK_BOT_TOKEN"
# ).to_string(),
"SSM_SLACK_SIGNING_SECRET":
self.ssm_signing_secret.parameter_name,
"SSM_SLACK_BOT_TOKEN":
self.ssm_bot_token.parameter_name,
"UPDATE_STATEMACHINE_ARN":
self.update_note_statemachine.state_machine_arn,
"UNFURL_STATEMACHINE_ARN":
self.unfurl_statemachine.state_machine_arn,
},
log_retention=logs.RetentionDays.FIVE_DAYS,
timeout=core.Duration.seconds(600),
)
self.apigw = apigateway.LambdaRestApi(self,
"BoltRestGw",
handler=self.bolt_lambda)
self.ssm_kibela_team.grant_read(self.step_lambda)
self.ssm_kibela_token.grant_read(self.step_lambda)
self.public_s3.grant_read_write(self.step_lambda)
self.public_s3.grant_delete(self.step_lambda)
self.private_s3.grant_read_write(self.step_lambda)
self.private_s3.grant_delete(self.step_lambda)
self.bolt_lambda.add_to_role_policy(
iam.PolicyStatement(resources=["*"],
actions=["lambda:InvokeFunction"]))
self.ssm_signing_secret.grant_read(self.bolt_lambda)
self.ssm_bot_token.grant_read(self.bolt_lambda)
self.update_note_statemachine.grant_start_execution(self.bolt_lambda)
self.unfurl_statemachine.grant(self.bolt_lambda,
"states:StartSyncExecution")
def __init__(self,
scope: core.Construct,
id: str,
resources: FsiSharedResources,
subnet_group_name: str = 'Default',
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Configure the container resources...
self.repo = assets.DockerImageAsset(self,
'Repo',
directory='src/fsi/earnings',
file='Dockerfile')
code = lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1])
# Configure security policies...
role = iam.Role(
self,
'Role',
assumed_by=iam.ServicePrincipal(service='lambda'),
description='HomeNet-{}-Fsi-EarningsReport'.format(
resources.landing_zone.zone_name),
role_name='fsi-earnings@homenet.{}.{}'.format(
resources.landing_zone.zone_name,
core.Stack.of(self).region).lower(),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name=
'service-role/AWSLambdaVPCAccessExecutionRole'),
])
# Grant any permissions...
self.earnings_table = d.Table(
self,
'EarningCalendar',
table_name='FsiCoreSvc-EarningsCalendar',
billing_mode=d.BillingMode.PAY_PER_REQUEST,
partition_key=d.Attribute(name='PartitionKey',
type=d.AttributeType.STRING),
sort_key=d.Attribute(name='SortKey', type=d.AttributeType.STRING),
time_to_live_attribute='Expiration',
point_in_time_recovery=True,
server_side_encryption=True)
self.earnings_table.grant_read_write_data(role)
# Define any variables for the function
self.function_env = {
'CACHE_TABLE': self.earnings_table.table_name,
}
# Create the backing webapi compute ...
self.function = lambda_.DockerImageFunction(
self,
'Function',
code=code,
role=role,
function_name='HomeNet-{}-Fsi-{}'.format(
resources.landing_zone.zone_name, FsiEarningsGateway.__name__),
description='Python Lambda function for ' +
FsiEarningsGateway.__name__,
timeout=core.Duration.seconds(30),
tracing=lambda_.Tracing.ACTIVE,
vpc=resources.landing_zone.vpc,
log_retention=logs.RetentionDays.FIVE_DAYS,
memory_size=128,
allow_all_outbound=True,
vpc_subnets=ec2.SubnetSelection(
subnet_group_name=subnet_group_name),
security_groups=[resources.landing_zone.security_group],
environment=self.function_env,
)
# Bind APIG to Lambda compute...
self.frontend_proxy = a.LambdaRestApi(
self,
'ApiGateway',
proxy=True,
handler=self.function,
options=a.RestApiProps(
description='Hosts the Earnings Calendar Services via ' +
self.function.function_name,
domain_name=a.DomainNameOptions(
domain_name='earnings.trader.fsi',
certificate=Certificate.from_certificate_arn(
self,
'Certificate',
certificate_arn=
'arn:aws:acm:us-east-2:581361757134:certificate/4e3235f7-49a1-42a5-a671-f2449b45f72d'
),
security_policy=a.SecurityPolicy.TLS_1_0),
policy=iam.PolicyDocument(statements=[
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=['execute-api:Invoke'],
principals=[iam.AnyPrincipal()],
resources=['*'],
conditions={
'IpAddress': {
'aws:SourceIp': [
'10.0.0.0/8',
'192.168.0.0/16',
'72.90.160.65/32'
]
}
})
]),
endpoint_configuration=a.EndpointConfiguration(
types=[a.EndpointType.REGIONAL], )))
# Register Dns Name
r53.ARecord(self,
'AliasRecord',
zone=resources.trader_dns_zone,
record_name='earnings.%s' %
resources.trader_dns_zone.zone_name,
target=r53.RecordTarget.from_alias(
dns_targets.ApiGateway(self.frontend_proxy)))
def __init__(self, scope: core.Construct, id: builtins.str, resources:FsiSharedResources, subnet_group_name:str='Default') -> None:
super().__init__(scope, id)
self.__resources = resources
self.datastores = FsiCollectionDataStoreConstruct(self,'DataStores',
resources=self.resources,
subnet_group_name=subnet_group_name)
self.eventing = FsiCollectionsEventing(self,'Eventing',
landing_zone=resources.landing_zone)
# Configure role...
role = iam.Role(self,'Role',
assumed_by=iam.ServicePrincipal(service='lambda'),
description='Ameritrade Data Collection Lambda via '+self.component_name,
role_name='{}@homenet-{}.{}'.format(
self.component_name,
resources.landing_zone.zone_name,
core.Stack.of(self).region),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name='service-role/AWSLambdaVPCAccessExecutionRole')
])
resources.tda_secret.grant_read(role)
self.datastores.instrument_table.grant_read_write_data(role)
self.datastores.transaction_table.grant_read_write_data(role)
self.datastores.quotes_table.grant_read_write_data(role)
self.datastores.options_table.grant_read_write_data(role)
# Configure the lambda...
self.repo = assets.DockerImageAsset(self,'Repo',
directory=source_directory)
code = lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1])
self.function = lambda_.DockerImageFunction(self,'Function',
code = code,
role= role,
function_name='HomeNet-Fsi{}-{}'.format(
resources.landing_zone.zone_name,
self.component_name),
description='Python container function for '+self.component_name,
timeout= core.Duration.minutes(15),
tracing= lambda_.Tracing.ACTIVE,
vpc= resources.landing_zone.vpc,
log_retention= RetentionDays.TWO_WEEKS,
memory_size= 3 * 128,
allow_all_outbound=True,
vpc_subnets=ec2.SubnetSelection(subnet_group_name=subnet_group_name),
security_groups=[resources.landing_zone.security_group],
environment={
'REGION':core.Stack.of(self).region,
'TDA_SECRET_ID': resources.tda_secret.secret_arn,
'TDA_REDIRECT_URI': ssm.StringParameter.from_string_parameter_name(self,'TDA_REDIRECT_URI',
string_parameter_name='/HomeNet/Amertitrade/redirect_uri').string_value,
'TDA_CLIENT_ID': ssm.StringParameter.from_string_parameter_name(self, 'TDA_CLIENT_ID',
string_parameter_name='/HomeNet/Ameritrade/client_id').string_value,
'INSTRUMENT_TABLE_NAME': self.datastores.instrument_table.table_name,
'TRANSACTION_TABLE_NAME': self.datastores.transaction_table.table_name,
'QUOTES_TABLE_NAME': self.datastores.quotes_table.table_name,
'OPTIONS_TABLE_NAME': self.datastores.options_table.table_name,
}
)
# Define the execution schedule...
self.add_states_schedule('DiscoverInstruments',
schedule=events.Schedule.cron(week_day='SUN',hour="0", minute="0"))
self.add_states_schedule('DiscoverOptionable',
schedule=events.Schedule.cron(week_day='SUN',hour="1", minute="0"))
self.add_states_schedule('CollectFundamentals',
schedule=events.Schedule.cron(week_day='SUN',hour="2", minute="0"))
self.add_states_schedule('CollectIntraday',
schedule=events.Schedule.cron(week_day='MON-FRI',hour="13-23/3", minute="0"),
payload={
'Action': 'CollectHistoric',
'CandleConfiguration':{
'period_type':'day',
'period':'1',
'frequency_type':'minute',
'frequency':'1'
}
})
self.add_states_schedule('CollectOptions',
schedule=events.Schedule.cron(week_day='MON-FRI',hour="12-23/3", minute="0"))
self.add_states_schedule('CollectClosingBell',
schedule=events.Schedule.cron(week_day='MON-FRI',hour="23", minute="0"),
payload={
'Action': 'CollectHistoric',
'CandleConfiguration':{
'period_type':'day',
'period':'1',
'frequency_type':'minute',
'frequency':'1'
}
})
self.add_states_schedule('CollectHistoric',
schedule=events.Schedule.cron(week_day='SAT',hour="0", minute="0"),
payload={
'Action': 'CollectHistoric',
'CandleConfiguration':{
'period_type':'year',
'period':'20',
'frequency_type':'daily',
'frequency':'1'
}
})
self.add_states_schedule('CollectTransactions',
schedule=events.Schedule.cron(week_day='SUN-FRI', minute="30"))
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# ==================================================
# =============== CFN PARAMETERS ===================
# ==================================================
project_name = core.CfnParameter(scope=self,
id='SageMakerProjectName',
type='String')
model_execution_role_arn = core.CfnParameter(
scope=self, id='ModelExecutionRoleArn', type='String')
model_binary_location = core.CfnParameter(scope=self,
id='ModelBinaryLocation',
type='String')
stage_name = core.CfnParameter(scope=self,
id='StageName',
type='String')
name = f'{project_name.value_as_string}-{stage_name.value_as_string}'
# ==================================================
# ================== IAM ROLE ======================
# ==================================================
role = iam.Role.from_role_arn(
scope=self,
id='role',
role_arn=model_execution_role_arn.value_as_string)
# ==================================================
# ================== ECR IMAGE =====================
# ==================================================
ecr_repository = ecr.Repository.from_repository_name(
scope=self,
id='repo',
repository_name='<ADD YOUR CONTAINER REPO HERE>')
ecr_image = aws_lambda.DockerImageCode.from_ecr(
repository=ecr_repository, tag='<ADD YOUR IMAGE TAG HERE>')
# ==================================================
# ================ LAMBDA FUNCTION =================
# ==================================================
lambda_function = aws_lambda.DockerImageFunction(
scope=self,
id='lambda',
function_name=name,
code=ecr_image,
memory_size=1024,
role=role,
environment={
'MODEL_S3_URI': model_binary_location.value_as_string,
},
timeout=core.Duration.seconds(60))
# ==================================================
# ================== API GATEWAY ===================
# ==================================================
api = apigw.HttpApi(scope=self,
id='api_gateway',
api_name=name,
cors_preflight={
"allow_headers": ["Authorization"],
"allow_methods": [apigw.HttpMethod.POST],
"allow_origins": ["*"],
"max_age": core.Duration.days(10)
})
integration = apigw.CfnIntegration(
scope=self,
id='integration',
api_id=api.http_api_id,
credentials_arn=role.role_arn,
integration_type='AWS_PROXY',
integration_uri=lambda_function.function_arn,
integration_method='POST',
payload_format_version='2.0')
apigw.CfnRoute(scope=self,
id='route',
api_id=api.http_api_id,
route_key='POST /',
target=f'integrations/{integration.ref}')
def __init__(self,
scope: core.Construct,
id: str,
infra: RtspBaseResourcesConstruct,
subnet_group_name: str = 'Default',
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
core.Tags.of(self).add(key='Source', value=PhotosApiConstruct.__name__)
# Configure the container resources...
self.repo = assets.DockerImageAsset(self,
'Repo',
directory='src/rtsp/photo-api',
file='Dockerfile')
code = lambda_.DockerImageCode.from_ecr(
repository=self.repo.repository,
tag=self.repo.image_uri.split(':')[-1])
# Configure security policies...
role = iam.Role(
self,
'Role',
assumed_by=iam.ServicePrincipal(service='lambda'),
description='HomeNet-{}-PhotoApi'.format(
infra.landing_zone.zone_name),
role_name='rtsp-photoapi@homenet.{}.{}'.format(
infra.landing_zone.zone_name,
core.Stack.of(self).region),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name=
'service-role/AWSLambdaVPCAccessExecutionRole'),
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name='AmazonS3ReadOnlyAccess')
])
infra.bucket.grant_read(role)
infra.face_table.grant_read_write_data(role)
# Define any variables for the function
self.function_env = {
'FACE_TABLE': infra.face_table.table_name,
'REGION': core.Stack.of(self).region,
}
# Create the backing webapi compute ...
self.function = lambda_.DockerImageFunction(
self,
'Function',
code=code,
role=role,
function_name='HomeNet-PhotoApi',
description='Python Lambda function for ' +
PhotosApiConstruct.__name__,
timeout=core.Duration.seconds(30),
tracing=lambda_.Tracing.ACTIVE,
vpc=infra.landing_zone.vpc,
log_retention=RetentionDays.FIVE_DAYS,
memory_size=128,
allow_all_outbound=True,
vpc_subnets=ec2.SubnetSelection(
subnet_group_name=subnet_group_name),
security_groups=[infra.security_group],
environment=self.function_env,
)
# Bind APIG to Lambda compute...
# Calls need to use https://photos-api.virtual.world
self.frontend_proxy = a.LambdaRestApi(
self,
'ApiGateway',
proxy=True,
handler=self.function,
options=a.RestApiProps(
description='Photo-Api proxy for ' +
self.function.function_name,
binary_media_types=['image/png', 'image/jpg', 'image/bmp'],
domain_name=a.DomainNameOptions(
domain_name='photos-api.virtual.world',
certificate=Certificate.from_certificate_arn(
self,
'Certificate',
certificate_arn=
'arn:aws:acm:us-east-1:581361757134:certificate/c91263e7-882e-441d-aa2f-717074aed6d0'
),
security_policy=a.SecurityPolicy.TLS_1_0),
policy=iam.PolicyDocument(statements=[
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=['execute-api:Invoke'],
principals=[iam.AnyPrincipal()],
resources=['*'],
conditions={
'IpAddress': {
'aws:SourceIp': [
'10.0.0.0/8',
'192.168.0.0/16',
'72.90.160.65/32'
]
}
})
]),
endpoint_configuration=a.EndpointConfiguration(
types=[a.EndpointType.REGIONAL],
#vpc_endpoints=[
# infra.landing_zone.vpc_endpoints.interfaces['execute-api']
#]
)))
def __init__(self, scope: core.Construct, construct_id: str, name: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
"""VPC - used in project"""
vpc = ec2.Vpc(self, f'{name}-VPC', max_azs=2)
"""Filesystem - shared between Lambda and Streamlit - Deletes when stack gets shut down"""
fs = efs.FileSystem(self,
f'{name}-FileSystem',
vpc=vpc,
removal_policy=core.RemovalPolicy.DESTROY)
access_point = fs.add_access_point(
'AccessPoint',
create_acl=efs.Acl(owner_gid='1001',
owner_uid='1001',
permissions='750'),
path="/export/lambda",
posix_user=efs.PosixUser(gid="1001", uid="1001"))
"""Model folder that contains Lambda code"""
model_folder = os.path.dirname(
os.path.realpath(__file__)) + "/../model"
lambda_handler = _lambda.DockerImageFunction(
self,
f'{name}-Lambda',
code=_lambda.DockerImageCode.from_image_asset(
model_folder), #Uses local code to build the container
memory_size=1024, #Adjust to your need - 128MB to 10GB
timeout=core.Duration.minutes(
5), #Adjust to your need - up to 15 mins
vpc=vpc,
filesystem=_lambda.FileSystem.from_efs_access_point(
access_point, MOUNT_POINT))
"""Custom Log groups for Lambda"""
lambda_lgs = logs.LogGroup(
self,
f'{name}-Lambda-LogGroup',
log_group_name=f"/aws/lambda/{lambda_handler.function_name}",
retention=logs.RetentionDays.ONE_WEEK,
removal_policy=core.RemovalPolicy.DESTROY)
"""API Gateway - integrates all methods and ressources - used for Lambda invocation"""
api = api_gw.HttpApi(
self,
f'{name}-ApiGw',
default_integration=integrations.LambdaProxyIntegration(
handler=lambda_handler))
"""""" """""" """""" """""" """""" """""" """""" """""" """"""
#STREAMLIT RELATED START
"""""" """""" """""" """""" """""" """""" """""" """""" """"""
'''
cluster = ecs.Cluster(self, f"{name}-Streamlit-Cluster", vpc=vpc)
ecs_task = ecs.FargateTaskDefinition(
self,
f'{name}-Streamlit-Task-Def',
)
streamlit_container = ecs_task.add_container(
f'{name}-Streamlit-Container',
image=ecs.ContainerImage.from_asset('streamlit-docker'),
essential=True,
environment={
'API_URL': api.url,
},
logging=ecs.LogDrivers.aws_logs(
stream_prefix=f'{name}-Streamlit-Log'
)
)
streamlit_container.add_port_mappings(
ecs.PortMapping(
container_port=8501,
host_port=8501,
protocol=ecs.Protocol.TCP
)
)
"""Efs Volume - shared between Lambda / Streamlit"""
ecs_task.add_volume(name=f'{name}-Efs-Volume',
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=fs.file_system_id,
))
"""Efs Mountpoint"""
streamlit_container.add_mount_points(
ecs.MountPoint(
container_path="/mnt/data",
read_only=False,
source_volume=f'{name}-Efs-Volume'
))
ecs_task.add_to_task_role_policy(
statement=iam.PolicyStatement(
actions=["efs:*"],
resources=['*'],
effect=iam.Effect.ALLOW
)
)
"""Fargate Service that hosts the Streamlit Application"""
ecs_service = ecs_patterns.ApplicationLoadBalancedFargateService(self, f'{name}-Fargate-Service',
cluster=cluster,
cpu=256,
desired_count=1,
task_definition = ecs_task,
memory_limit_mib=512,
public_load_balancer=True,
platform_version=ecs.FargatePlatformVersion.VERSION1_4, #https://forums.aws.amazon.com/thread.jspa?messageID=960420
)
fs.connections.allow_default_port_from(
ecs_service.service.connections)
'''
"""""" """""" """""" """""" """""" """""" """""" """""" """"""
#STREAMLIT RELATED END
"""""" """""" """""" """""" """""" """""" """""" """""" """"""
core.CfnOutput(self, 'URL', value=api.url)