def __init__(self, app: App, id: str) -> None:
super().__init__(app, id)
with open("lambda-handler.py", encoding="utf8") as fp:
handler_code = fp.read()
# Creates reference to already existing kinesis stream
kinesis_stream = kinesis.Stream.from_stream_arn(
self, 'KinesisStream',
Arn.format(
ArnComponents(resource='stream',
service='kinesis',
resource_name='my-stream'), self))
lambdaFn = lambda_.Function(self,
'Singleton',
handler='index.main',
code=lambda_.InlineCode(handler_code),
runtime=lambda_.Runtime.PYTHON_3_7,
timeout=Duration.seconds(300))
# Update Lambda Permissions To Use Stream
kinesis_stream.grant_read(lambdaFn)
# Create New Kinesis Event Source
kinesis_event_source = event_sources.KinesisEventSource(
stream=kinesis_stream,
starting_position=lambda_.StartingPosition.LATEST,
batch_size=1)
# Attach New Event Source To Lambda
lambdaFn.add_event_source(kinesis_event_source)
def create_stream_processor(self):
# API Endpoint
stream_function_source = _lmbd.Code.asset("../stream_function")
# Stream processor lambda function
stream_function = _lmbd.Function(
self,
"stream-function",
handler="main.handler",
function_name=f"{self.prefix}-stream-function",
runtime=_lmbd.Runtime.PYTHON_3_8,
code=stream_function_source,
timeout=core.Duration.seconds(5),
memory_size=128,
environment={
"OUTPUT_BUCKET": self.s3_bucket.bucket_name,
},
)
# Enable access to S3 bucket
self.s3_bucket.grant_read_write(stream_function)
# Connecting the processor to the incoming event stream
self.stream.grant_read(stream_function)
speev_event_source = _srcs.KinesisEventSource(
stream=self.stream,
starting_position=_lmbd.StartingPosition.LATEST,
batch_size=1000,
max_batching_window=core.Duration.seconds(10),
)
stream_function.add_event_source(speev_event_source)
def __init__(self, scope: core.Construct, id: str, stream: ks.Stream,
auditTable: ddb.Table, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.kinesis_processor = lambda_.Function(
self,
'KinesisProcessor',
handler=
'EmailServiceLambda::FunctionHandler.HandleKinesisEventAsync',
code=lambda_.Code.from_asset(path='../bin/EmailServiceLambda.zip'),
timeout=core.Duration.minutes(1),
tracing=lambda_.Tracing.ACTIVE,
runtime=lambda_.Runtime.DOTNET_CORE_2_1,
environment={
'TABLE_NAME_PREFIX': auditTable.table_name,
'EMAIL_TO': 'no-where@null'
})
for policy in [
'AmazonSESFullAccess', 'AmazonDynamoDBFullAccess',
'AWSXrayWriteOnlyAccess'
]:
self.kinesis_processor.role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(policy))
self.kinesis_processor.add_event_source(
sources.KinesisEventSource(
stream=stream,
starting_position=lambda_.StartingPosition.LATEST,
batch_size=10))
def __configure_ingestion(self) -> None:
self.__updates_stream = k.Stream(
self,
'PortfolioUpdates',
encryption=k.StreamEncryption.MANAGED,
retention_period=core.Duration.days(7),
shard_count=1,
stream_name='portfolio-updates')
self.__updates_handler = PythonLambda(
self,
'UpdatesHandler',
build_prefix='artifacts/FinSurf-PortfolioMgmt-UpdatesHandler',
handler='updates_handler.lambda_handler',
subnet_group_name='PortfolioMgmt',
context=self.context,
securityGroups=[self.security_group]).function
self.updates_handler.add_event_source(source=evt.KinesisEventSource(
stream=self.updates_stream,
starting_position=lambda_.StartingPosition.LATEST))
# Configure writing to neptune
self.updates_handler.add_environment(
key='NEPTUNE_ENDPOINT', value=self.neptune_cluster.attr_endpoint)
def __init__(self, scope: core.Construct, id: str, *, prefix: str,
environment: str, configuration, **kwargs):
"""
:param scope: Stack class, used by CDK.
:param id: ID of the construct, used by CDK.
:param prefix: Prefix of the construct, used for naming purposes.
:param environment: Environment of the construct, used for naming purposes.
:param configuration: Configuration of the construct. In this case IOT_SNS_CONFIG_SCHEMA.
:param kwargs: Other parameters that could be used by the construct.
"""
super().__init__(scope, id, **kwargs)
self.prefix = prefix
self.environment_ = environment
self._configuration = configuration
# Validating that the payload passed is correct
validate_configuration(
configuration_schema=IOT_KINESIS_FIREHOSE_CONFIG_SCHEMA,
configuration_received=self._configuration)
# Defining Kinesis Stream
stream_data = deepcopy(self._configuration["stream"])
self._kinesis_stream = base_kinesis_firehose_delivery_stream(
self, **stream_data)
# Defining IAM Role
role = base_kinesis_role(self,
resource_name=stream_data["stream_name"],
principal_resource="iot")
# Validating Lambda Function Runtime
functions_data = self._configuration["lambda_handlers"]
self._lambda_functions = list()
for setting in functions_data:
_lambda_function = base_lambda_function(
self, **setting["lambda_handler"])
self._lambda_functions.append(_lambda_function)
# Defining Function Subscription
event_source = event_src.KinesisEventSource(
stream=self._kinesis_stream, **setting["event_settings"])
_lambda_function.add_event_source(event_source)
# Defining Topic Rule properties
action = iot.CfnTopicRule.KinesisActionProperty(
stream_name=self._kinesis_stream.stream_name,
role_arn=role.role_arn)
action_property = iot.CfnTopicRule.ActionProperty(kinesis=action)
rule_data = self._configuration["iot_rule"]
self._iot_rule = base_iot_rule(self,
action_property=action_property,
**rule_data)
def __configure_ingestion(self)->None:
self.__updates_handler = PythonLambda(
self,'FinSurf-GraphBuilder',
build_prefix='artifacts/FinSurf-GraphBuilder',
handler='handlers.kinesis_event_handler',
subnet_group_name='MarketGraph',
context=self.context,
securityGroups= [self.security_group]).function
self.updates_handler.add_event_source(
source=evt.KinesisEventSource(
stream= self.quotes_stream,
starting_position=lambda_.StartingPosition.TRIM_HORIZON))
self.updates_handler.add_event_source(
source=evt.KinesisEventSource(
stream= self.fundamental_stream,
starting_position=lambda_.StartingPosition.TRIM_HORIZON))
# Configure writing to neptune
self.updates_handler.add_environment(
key='NEPTUNE_ENDPOINT', value=self.neptune_cluster.attr_endpoint)
def __init__(self, app: core.App, id: str) -> None:
super().__init__(app, id)
with open("lambda-handler.py", encoding="utf8") as fp:
handler_code = fp.read()
Kstream = kinesis_.Stream(self,
"KinesisSagemakerInference",
encryption=None,
encryption_key=None,
retention_period_hours=24,
shard_count=1)
lambdaFn = lambda_.Function(
self,
"KinesisSMLambda",
code=lambda_.InlineCode(handler_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={
"endpoint_name":
endpoint_name, # CHANGE TO YOUR ENDPOINT NAME!!
"content_type": "text/csv",
"input_data": input_data,
"bucket": bucket,
"key": key
})
lambdaFn.add_to_role_policy(
aws_iam.PolicyStatement(
actions=[
'sagemaker:InvokeEndpoint',
],
resources=[
'arn:aws:sagemaker:{}:{}:endpoint/{}'.format(
my_region, my_acc_id, endpoint_name),
]))
# Add the Kinesis stream as Lambda source
lambdaFn.add_event_source(
aws_lambda_event_sources.KinesisEventSource(
Kstream, starting_position=lambda_.StartingPosition.LATEST))
# Add stream read permissions
Kstream.grant_read(lambdaFn.role)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
table = aws_dynamodb.Table(self, "DashboardModel",
partition_key=aws_dynamodb.Attribute(name="Pk", type=aws_dynamodb.AttributeType.STRING),
sort_key=aws_dynamodb.Attribute(name="Sk", type=aws_dynamodb.AttributeType.STRING),
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST)
kds_input_stream = aws_kinesis.Stream(self, "kds_dashboard_input_stream",
shard_count=1,
stream_name="kds_dashboard_input_stream")
kds_output_stream = aws_kinesis.Stream(self, "kds_dashboard_output_stream",
shard_count=1,
stream_name="kds_dashboard_output_stream")
# Creating a ingest bucket for this stack
ingest_bucket = aws_s3.Bucket(self,'dreis_dboard_ingest_bucket')
kfh_service_role = aws_iam.Role(self, 'KFH_Dashboard_Role',
assumed_by=aws_iam.ServicePrincipal('firehose.amazonaws.com')
)
kfh_policy_stmt = aws_iam.PolicyStatement(
actions=["*"],
resources=["*"]
)
kfh_service_role.add_to_policy(kfh_policy_stmt)
#Creating firehose for this stack
kfh_source = aws_kinesisfirehose.CfnDeliveryStream.KinesisStreamSourceConfigurationProperty(
kinesis_stream_arn=kds_input_stream.stream_arn,
role_arn=kfh_service_role.role_arn
)
kfh_datalake = aws_kinesisfirehose.CfnDeliveryStream(self, "kfh_datalake",
s3_destination_configuration=aws_kinesisfirehose.CfnDeliveryStream.S3DestinationConfigurationProperty(
bucket_arn=ingest_bucket.bucket_arn,
buffering_hints=aws_kinesisfirehose.CfnDeliveryStream.BufferingHintsProperty(
interval_in_seconds=60,
size_in_m_bs=5),
compression_format="UNCOMPRESSED",
role_arn=kfh_service_role.role_arn
),
delivery_stream_type="KinesisStreamAsSource",
kinesis_stream_source_configuration=kfh_source
)
kda_service_role = aws_iam.Role(self, 'KDA_Dashboard_Role',
assumed_by=aws_iam.ServicePrincipal('kinesisanalytics.amazonaws.com')
)
kda_policy_stmt = aws_iam.PolicyStatement(
actions=["*"],
resources=["*"]
)
kda_service_role.add_to_policy(kda_policy_stmt)
# KA doesn't like - (dash) in names
col1 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="state",
sql_type="VARCHAR(2)",
mapping="$.state"
)
col2 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="event_time",
sql_type="TIMESTAMP",
mapping="$.event-time"
)
col3 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="region",
sql_type="VARCHAR(12)",
mapping="$.region"
)
col4 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="store_id",
sql_type="INTEGER",
mapping="$.store-id"
)
col5 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="kpi_1",
sql_type="INTEGER",
mapping="$.kpi-1"
)
col6 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="kpi_2",
sql_type="INTEGER",
mapping="$.kpi-2"
)
col7 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="kpi_3",
sql_type="INTEGER",
mapping="$.kpi-3"
)
col8 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="kpi_4",
sql_type="INTEGER",
mapping="$.kpi-4"
)
col9 = aws_kinesisanalytics.CfnApplication.RecordColumnProperty(
name="kpi_5",
sql_type="INTEGER",
mapping="$.kpi-5"
)
schema = aws_kinesisanalytics.CfnApplication.InputSchemaProperty(
record_columns=[col2, col1, col3, col4, col5, col6, col7, col8, col9],
record_encoding="UTF-8",
record_format=aws_kinesisanalytics.CfnApplication.RecordFormatProperty(
record_format_type="JSON",
mapping_parameters=aws_kinesisanalytics.CfnApplication.MappingParametersProperty(
json_mapping_parameters=aws_kinesisanalytics.CfnApplication.JSONMappingParametersProperty(
record_row_path="$"
)
)
)
)
kda_is = aws_kinesisanalytics.CfnApplication.KinesisStreamsInputProperty(
resource_arn=kds_input_stream.stream_arn,
role_arn=kda_service_role.role_arn
)
ip = aws_kinesisanalytics.CfnApplication.InputProperty(
name_prefix="SOURCE_SQL_STREAM",
input_schema=schema,
kinesis_streams_input=kda_is
)
application_code = "CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM_BY_STORE\" (\"region\" VARCHAR(10), \"state\" VARCHAR(2), \"store-id\" INTEGER, kpi_1_sum INTEGER, kpi_2_sum INTEGER, ingest_time TIMESTAMP);" + \
"CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM_BY_STATE\" (\"region\" VARCHAR(10), \"state\" VARCHAR(2), kpi_1_sum INTEGER, kpi_2_sum INTEGER, ingest_time TIMESTAMP);" + \
"CREATE OR REPLACE STREAM \"DESTINATION_SQL_STREAM_BY_REGION\" (\"region\" VARCHAR(10), kpi_1_sum INTEGER, kpi_2_sum INTEGER, ingest_time TIMESTAMP);" + \
"CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM_BY_STORE\"" + \
"SELECT STREAM \"region\", \"state\", \"store-id\", SUM(\"kpi-1\") AS kpi_1_sum, SUM(\"kpi-2\") AS kpi_2_sum, FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE) as ingest_time" + \
"FROM \"SOURCE_SQL_STREAM_001\"" + \
"GROUP BY \"region\", \"state\", \"store-id\", FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE), FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);" + \
"CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM_BY_STATE\"" + \
"SELECT STREAM \"region\", \"state\", SUM(\"kpi-1\") AS kpi_1_sum, SUM(\"kpi-2\") AS kpi_2_sum, FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE) as ingest_time" + \
"FROM \"SOURCE_SQL_STREAM_001\"" + \
"GROUP BY \"region\", \"state\", FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE), FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);" + \
"CREATE OR REPLACE PUMP \"STREAM_PUMP\" AS INSERT INTO \"DESTINATION_SQL_STREAM_BY_REGION\"" + \
"SELECT STREAM \"region\", SUM(\"kpi-1\") AS kpi_1_sum, SUM(\"kpi-2\") AS kpi_2_sum, FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE) as ingest_time" + \
"FROM \"SOURCE_SQL_STREAM_001\"" + \
"GROUP BY \"region\", FLOOR(\"SOURCE_SQL_STREAM_001\".APPROXIMATE_ARRIVAL_TIME TO MINUTE), FLOOR((\"SOURCE_SQL_STREAM_001\".ROWTIME - TIMESTAMP '1970-01-01 00:00:00') SECOND / 10 TO SECOND);"
kda_app = aws_kinesisanalytics.CfnApplication(self, "kda_agg",
inputs=[ip], #kda_inputs,
application_code=application_code,
application_description="Aggregating data",
application_name="DashboardMetricsAggregator"
)
kda_output_prop = aws_kinesisanalytics.CfnApplicationOutput.KinesisStreamsOutputProperty(
resource_arn=kds_output_stream.stream_arn,
role_arn=kda_service_role.role_arn
)
kda_dest_schema = aws_kinesisanalytics.CfnApplicationOutput.DestinationSchemaProperty(
record_format_type="JSON"
)
kda_output_prop_by_store = aws_kinesisanalytics.CfnApplicationOutput.OutputProperty(
destination_schema=kda_dest_schema,
kinesis_streams_output=kda_output_prop,
name="DESTINATION_SQL_STREAM_BY_STORE"
)
kda_output_prop_by_state = aws_kinesisanalytics.CfnApplicationOutput.OutputProperty(
destination_schema=kda_dest_schema,
kinesis_streams_output=kda_output_prop,
name="DESTINATION_SQL_STREAM_BY_STATE"
)
kda_output_prop_by_region = aws_kinesisanalytics.CfnApplicationOutput.OutputProperty(
destination_schema=kda_dest_schema,
kinesis_streams_output=kda_output_prop,
name="DESTINATION_SQL_STREAM_BY_REGION"
)
kda_app_output_prop = aws_kinesisanalytics.CfnApplicationOutput(self, "kda_agg_output_store",
application_name="DashboardMetricsAggregator",
output=kda_output_prop_by_store
)
kda_app_output_prop = aws_kinesisanalytics.CfnApplicationOutput(self, "kda_agg_output_state",
application_name="DashboardMetricsAggregator",
output=kda_output_prop_by_state
)
kda_app_output_prop = aws_kinesisanalytics.CfnApplicationOutput(self, "kda_agg_output_region",
application_name="DashboardMetricsAggregator",
output=kda_output_prop_by_region
)
lambda_agg_function = aws_lambda.Function(self, "AggDataLambda",
runtime=aws_lambda.Runtime.PYTHON_3_7,
handler="lambda_function.lambda_handler",
code=aws_lambda.Code.asset("../models/dashboard/lambdas/aggregate_data_lambda"),
timeout=Duration.minutes(5))
lambda_agg_function.add_environment("DDB_TABLE_DASHBOARD", table.table_name)
lambda_agg_function.add_to_role_policy(aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"kinesis:*"
],
resources=["*"]
))
table.grant_read_write_data(lambda_agg_function)
kes = aws_lambda_event_sources.KinesisEventSource(kds_output_stream,
starting_position=aws_lambda.StartingPosition.TRIM_HORIZON,
batch_size=50,
#max_batching_window=100
)
lambda_agg_function.add_event_source(kes)
core.CfnOutput(
self, "TableName_Dashboard",
description="Table name for Dashboard",
value=table.table_name
)
core.CfnOutput(
self, "BucketName_Dashboard",
description="Bucket name",
value=ingest_bucket.bucket_arn
)
core.CfnOutput(
self, "KinesisInputStream_Dashboard",
description="Kinesis input for Dashboard",
value=kds_input_stream.stream_name
)
core.CfnOutput(
self, "KinesisOutputStream_Dashboard",
description="Kinesis output for Dashboard",
value=kds_output_stream.stream_name
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Kinesis to lambda
self.stream_lambda = kinesis_lambda.KinesisStreamsToLambda(
self,
'clickstream',
lambda_function_props=_lambda.FunctionProps(
runtime=_lambda.Runtime.PYTHON_3_7,
handler='index.lambda_handler',
code=_lambda.Code.inline(
get_code('send_data_to_firehose.py'))),
kinesis_stream_props=kinesis.StreamProps(
stream_name='clickstream',
retention_period=core.Duration.days(1),
shard_count=4),
kinesis_event_source_props=lambda_sources.KinesisEventSourceProps(
starting_position=_lambda.StartingPosition.TRIM_HORIZON,
batch_size=1))
# Lambda to produce data
self.produce_fake_data = _lambda.Function(
self,
'produce_data',
runtime=_lambda.Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(90),
handler='index.lambda_handler',
code=_lambda.Code.inline(get_code('produce_data.py')),
environment={
'STREAM_NAME': self.stream_lambda.kinesis_stream.stream_name
})
self.stream_lambda.kinesis_stream.grant_read_write(
self.produce_fake_data)
# EventBridge to activate my function above
self.event_rule = events.Rule(
self,
'scheduledRule',
schedule=events.Schedule.expression('rate(1 minute)'))
self.event_rule.add_target(
targets.LambdaFunction(self.produce_fake_data))
# S3 Bucket
self.bucket = s3.Bucket(self,
'data-clicks-lake',
removal_policy=core.RemovalPolicy.DESTROY,
auto_delete_objects=True)
# Glue
self.glue_db_analytical = glue.Database(
self,
'analytic_clickstream',
database_name='clickstream_db',
location_uri=None,
)
self.glue_table_analytical = glue.Table(
self,
'analytical-table',
table_name='analytical-table',
columns=[
glue_column('custid', 'int'),
glue_column('trafficfrom', 'string'),
glue_column('url', 'string'),
glue_column('device', 'string'),
glue_column('touchproduct', 'int'),
glue_column('trans_timestamp', 'string')
],
database=self.glue_db_analytical,
data_format=glue.DataFormat.PARQUET,
bucket=self.bucket,
s3_prefix='kinesis/',
)
# Firehose
iam_role_firehose_analytical = self.create_firehose_role()
self.bucket.grant_read_write(iam_role_firehose_analytical)
firehose_props = FirehoseProps(
bucket=self.bucket,
role=iam_role_firehose_analytical,
stream=self.stream_lambda.kinesis_stream,
glue_db=self.glue_db_analytical,
glue_table=self.glue_table_analytical)
self.firehose = FirehoseLib(self, 'firehose_clickstream',
firehose_props)
# Elasticsearh
self.es_domain = ElasticsearchLib(self,
'ES-clickstream-domain').es_domain
# Lambda to send data to Elasticsearch
self.send_data_to_elasticsearch = lambda_python.PythonFunction(
self,
'clickstream_to_es',
entry='./analytics_ml_flow/lambda/lambda_with_requirements/',
handler='handler',
timeout=core.Duration.seconds(180),
index='Kinesis_ES.py',
environment={
'ES_HOST_HTTP': self.es_domain.domain_endpoint,
'ES_INDEX': 'clickstream',
'ES_IND_TYPE': 'transactions',
'ES_REGION': 'us-west-2',
})
self.es_domain.grant_index_read_write('clickstream',
self.send_data_to_elasticsearch)
self.es_domain.grant_read_write(self.send_data_to_elasticsearch)
stream_source = lambda_sources.KinesisEventSource(
self.stream_lambda.kinesis_stream,
starting_position=_lambda.StartingPosition.TRIM_HORIZON,
batch_size=1)
self.stream_lambda.kinesis_stream.grant_read(
self.send_data_to_elasticsearch)
self.send_data_to_elasticsearch.add_event_source(stream_source)
# Glue Crawler
crawler_role = self.create_crawler_permissions()
glue_props = GlueCrawlerProps(bucket=self.bucket, role=crawler_role)
self.glue_crawler = GlueCrawlerLib(self, 'glueCrawler', glue_props)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
#kinesis data stream
kinesis_stream = aws_kinesis.Stream(
self,
"kinesisStream",
retention_period=core.Duration.hours(24),
shard_count=1,
stream_name="kinesis_test_data_pipe")
#s3 for store stream data events
kinesis_s3_bucket = aws_s3.Bucket(
self, "kinesisS3Bucket", removal_policy=core.RemovalPolicy.DESTROY)
#Lambda functions
#import function codes - data consume
try:
with open("deployments/functions/stream_data_get.py",
mode="r") as file:
function_body_get = file.read()
except OSError:
print('File can not read')
#consume function
stream_get_function = aws_lambda.Function(
self,
"consumeFunction",
function_name="StreamConsumeFunction",
description="Process Data Streams and store to s3",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="index.lambda_handler",
code=aws_lambda.InlineCode(function_body_get),
timeout=core.Duration.seconds(5),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"BUCKET_NAME": f"{kinesis_s3_bucket.bucket_name}"
})
#permision to use stream by lambda
kinesis_stream.grant_read(stream_get_function)
#s3 permision
lambdas3Permision = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=[f"{kinesis_s3_bucket.bucket_arn}/*"],
actions=["s3:PutObject"])
lambdas3Permision.sid = "S3writePermisionToLambda"
stream_get_function.add_to_role_policy(lambdas3Permision)
#logs
stream_concume_logs = aws_logs.LogGroup(
self,
"StreamConcumeLogs",
log_group_name=f"/aws/lambda/{stream_get_function.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=aws_logs.RetentionDays.ONE_DAY)
#kinesis event source
kinesis_event_sources = aws_lambda_es.KinesisEventSource(
stream=kinesis_stream,
starting_position=aws_lambda.StartingPosition.LATEST,
batch_size=1)
#attached kinesis to lambda
stream_get_function.add_event_source(kinesis_event_sources)
#generate stream lambda function#####
#import function codes - data consume
try:
with open("deployments/functions/stream_data_gen.py",
mode="r") as file:
function_body_gen = file.read()
except OSError:
print('File can not read')
#stream generate function
stream_gen_function = aws_lambda.Function(
self,
"GenarateFunction",
function_name="StreamGenerateFunction",
description="Generate Data Streams",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="index.lambda_handler",
code=aws_lambda.InlineCode(function_body_gen),
timeout=core.Duration.seconds(60),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"STREAM_NAME": f"{kinesis_stream.stream_name}"
})
#permision to lambda to write into kinesis
kinesis_stream.grant_read_write(stream_gen_function)
#logs
stream_generate_logs = aws_logs.LogGroup(
self,
"StreamGenerateLogs",
log_group_name=f"/aws/lambda/{stream_gen_function.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=aws_logs.RetentionDays.ONE_DAY)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Add your stack resources below):
# Create Kinesis Data Stream
stream_data_pipe = _kinesis.Stream(
self,
"streamDataPipe",
retention_period=core.Duration.hours(24),
shard_count=1,
stream_name="data_pipe")
# Create an S3 Bucket for storing streaming data events
stream_data_store = _s3.Bucket(
self, "streamDataLake", removal_policy=core.RemovalPolicy.DESTROY)
# Read Lambda Code
try:
with open(
"advanced_use_cases/lambda_src/stream_record_consumer.py",
mode="r") as f:
stream_consumer_fn_code = f.read()
except OSError:
print("Unable to read lambda function code")
# Deploy the lambda function
stream_consumer_fn = _lambda.Function(
self,
"streamConsumerFn",
function_name="stream_consumer_fn",
description=
"Process streaming data events from kinesis and store in S3",
runtime=_lambda.Runtime.PYTHON_3_7,
handler="index.lambda_handler",
code=_lambda.InlineCode(stream_consumer_fn_code),
timeout=core.Duration.seconds(3),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"BUCKET_NAME": f"{stream_data_store.bucket_name}"
})
# Update Lambda Permissions To Use Stream
stream_data_pipe.grant_read(stream_consumer_fn)
# Add permissions to lambda to write to S3
roleStmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[f"{stream_data_store.bucket_arn}/*"],
actions=["s3:PutObject"])
roleStmt1.sid = "AllowLambdaToWriteToS3"
stream_consumer_fn.add_to_role_policy(roleStmt1)
# Create Custom Loggroup for Consumer
stream_consumer_lg = _logs.LogGroup(
self,
"streamConsumerLogGroup",
log_group_name=f"/aws/lambda/{stream_consumer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
# Create New Kinesis Event Source
stream_data_pipe_event_source = _lambda_event_sources.KinesisEventSource(
stream=stream_data_pipe,
starting_position=_lambda.StartingPosition.LATEST,
batch_size=1)
# Attach Kinesis Event Source To Lambda
stream_consumer_fn.add_event_source(stream_data_pipe_event_source)
########################################
####### #######
####### Stream Data Producer #######
####### #######
########################################
# Read Lambda Code
try:
with open("advanced_use_cases/lambda_src/stream_data_producer.py",
mode="r") as f:
data_producer_fn_code = f.read()
except OSError:
print("Unable to read lambda function code")
# Deploy the lambda function
data_producer_fn = _lambda.Function(
self,
"streamDataProducerFn",
function_name="data_producer_fn",
description=
"Produce streaming data events and push to Kinesis stream",
runtime=_lambda.Runtime.PYTHON_3_7,
handler="index.lambda_handler",
code=_lambda.InlineCode(data_producer_fn_code),
timeout=core.Duration.seconds(60),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"STREAM_NAME": f"{stream_data_pipe.stream_name}"
})
# Grant our Lambda Producer privileges to write to Kinesis Data Stream
stream_data_pipe.grant_read_write(data_producer_fn)
# Create Custom Loggroup for Producer
data_producer_lg = _logs.LogGroup(
self,
"dataProducerLogGroup",
log_group_name=f"/aws/lambda/{data_producer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
