def create_stream(stream_name, delivery_stream_type='DirectPut', delivery_stream_type_configuration=None,
s3_destination=None, elasticsearch_destination=None, tags=None, region_name=None):
tags = tags or {}
stream = {
'DeliveryStreamType': delivery_stream_type,
'KinesisStreamSourceConfiguration': delivery_stream_type_configuration,
'HasMoreDestinations': False,
'VersionId': '1',
'CreateTimestamp': time.time(),
'DeliveryStreamARN': firehose_stream_arn(stream_name),
'DeliveryStreamStatus': 'ACTIVE',
'DeliveryStreamName': stream_name,
'Destinations': [],
'Tags': tags
}
DELIVERY_STREAMS[stream_name] = stream
if elasticsearch_destination:
update_destination(stream_name=stream_name, destination_id=short_uid(),
elasticsearch_update=elasticsearch_destination)
if s3_destination:
update_destination(stream_name=stream_name, destination_id=short_uid(), s3_update=s3_destination)
# record event
event_publisher.fire_event(event_publisher.EVENT_FIREHOSE_CREATE_STREAM,
payload={'n': event_publisher.get_hash(stream_name)})
if delivery_stream_type == 'KinesisStreamAsSource':
kinesis_stream_name = delivery_stream_type_configuration.get('KinesisStreamARN').split('/')[1]
kinesis_connector.listen_to_kinesis(
stream_name=kinesis_stream_name, fh_d_stream=stream_name,
listener_func=process_records, wait_until_started=True,
ddb_lease_table_suffix='-firehose', region_name=region_name)
return stream
def test_run_kcl(self):
result = []
def process_records(records):
result.extend(records)
# start Kinesis client
stream_name = 'test-foobar'
aws_stack.create_kinesis_stream(stream_name, delete=True)
kinesis_connector.listen_to_kinesis(stream_name=stream_name,
listener_func=process_records,
kcl_log_level=logging.INFO,
wait_until_started=True)
kinesis = aws_stack.connect_to_service('kinesis')
stream_summary = kinesis.describe_stream_summary(
StreamName=stream_name)
self.assertEqual(
stream_summary['StreamDescriptionSummary']['OpenShardCount'], 1)
num_events_kinesis = 10
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'test_%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=stream_name)
def check_events():
self.assertEqual(len(result), num_events_kinesis)
retry(check_events, retries=4, sleep=2)
def create_stream(stream_name,
delivery_stream_type='DirectPut',
delivery_stream_type_configuration=None,
s3_destination=None,
elasticsearch_destination=None):
stream = {
'DeliveryStreamType': delivery_stream_type,
'KinesisStreamSourceConfiguration': delivery_stream_type_configuration,
'HasMoreDestinations': False,
'VersionId': '1',
'CreateTimestamp': time.time(),
'DeliveryStreamARN': firehose_stream_arn(stream_name),
'DeliveryStreamStatus': 'ACTIVE',
'DeliveryStreamName': stream_name,
'Destinations': []
}
DELIVERY_STREAMS[stream_name] = stream
if elasticsearch_destination:
update_destination(stream_name=stream_name,
destination_id=short_uid(),
elasticsearch_update=elasticsearch_destination)
if s3_destination:
update_destination(stream_name=stream_name,
destination_id=short_uid(),
s3_update=s3_destination)
if delivery_stream_type == 'KinesisStreamAsSource':
kinesis_stream_name = delivery_stream_type_configuration.get(
'KinesisStreamARN').split('/')[1]
kinesis_connector.listen_to_kinesis(stream_name=kinesis_stream_name,
fh_d_stream=stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix='-firehose')
return stream
def test_kinesis_lambda_ddb_streams(env=ENV_DEV):
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating stream...')
aws_stack.create_kinesis_stream(TEST_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
env=env, stream_view_type='NEW_AND_OLD_IMAGES')
# list streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
event_source_arn = stream['StreamArn']
assert event_source_arn
# deploy test lambda
script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py'))
zip_file = testutil.create_lambda_archive(script, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=event_source_arn)
# put items to table
num_events = 10
print('Putting %s items to table...' % num_events)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events):
table.put_item(Item={
PARTITION_KEY: 'testId123',
'data': 'foobar123'
})
print("Waiting some time before finishing test.")
time.sleep(5)
print('DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' % (len(EVENTS), num_events))
if len(EVENTS) != num_events:
print('ERROR receiving DynamoDB updates. Running processes:')
print(run("ps aux | grep 'python\|java\|node'"))
assert len(EVENTS) == num_events
def test_create_kinesis_event_source_mapping(self):
function_name = f"lambda_func-{short_uid()}"
stream_name = f"test-foobar-{short_uid()}"
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON_ECHO,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
arn = aws_stack.kinesis_stream_arn(stream_name,
account_id="000000000000")
lambda_client = aws_stack.create_external_boto_client("lambda")
lambda_client.create_event_source_mapping(EventSourceArn=arn,
FunctionName=function_name)
def process_records(record):
assert record
aws_stack.create_kinesis_stream(stream_name, delete=True)
kinesis_connector.listen_to_kinesis(
stream_name=stream_name,
listener_func=process_records,
wait_until_started=True,
)
kinesis = aws_stack.create_external_boto_client("kinesis")
stream_summary = kinesis.describe_stream_summary(
StreamName=stream_name)
self.assertEqual(
1, stream_summary["StreamDescriptionSummary"]["OpenShardCount"])
num_events_kinesis = 10
kinesis.put_records(
Records=[{
"Data": "{}",
"PartitionKey": "test_%s" % i
} for i in range(0, num_events_kinesis)],
StreamName=stream_name,
)
events = get_lambda_log_events(function_name)
self.assertEqual(10, len(events[0]["Records"]))
self.assertIn("eventID", events[0]["Records"][0])
self.assertIn("eventSourceARN", events[0]["Records"][0])
self.assertIn("eventSource", events[0]["Records"][0])
self.assertIn("eventVersion", events[0]["Records"][0])
self.assertIn("eventName", events[0]["Records"][0])
self.assertIn("invokeIdentityArn", events[0]["Records"][0])
self.assertIn("awsRegion", events[0]["Records"][0])
self.assertIn("kinesis", events[0]["Records"][0])
def run_kcl_with_iam_assume_role():
env_vars = {}
if os.environ.get('AWS_ASSUME_ROLE_ARN'):
env_vars['AWS_ASSUME_ROLE_ARN'] = os.environ.get('AWS_ASSUME_ROLE_ARN')
env_vars['AWS_ASSUME_ROLE_SESSION_NAME'] = os.environ.get(
'AWS_ASSUME_ROLE_SESSION_NAME')
env_vars['ENV'] = os.environ.get('ENV') or 'main'
def process_records(records):
print(records)
# start Kinesis client
stream_name = 'test-foobar'
kinesis_connector.listen_to_kinesis(stream_name=stream_name,
listener_func=process_records,
env_vars=env_vars,
kcl_log_level=logging.INFO,
wait_until_started=True)
def run_kcl_with_iam_assume_role():
env_vars = {}
if os.environ.get('AWS_ASSUME_ROLE_ARN'):
env_vars['AWS_ASSUME_ROLE_ARN'] = os.environ.get('AWS_ASSUME_ROLE_ARN')
env_vars['AWS_ASSUME_ROLE_SESSION_NAME'] = os.environ.get('AWS_ASSUME_ROLE_SESSION_NAME')
env_vars['ENV'] = os.environ.get('ENV') or 'main'
def process_records(records):
print(records)
# start Kinesis client
stream_name = 'test-foobar'
kinesis_connector.listen_to_kinesis(
stream_name=stream_name,
listener_func=process_records,
env_vars=env_vars,
kcl_log_level=logging.INFO,
wait_until_started=True)
def test_run_kcl_with_iam_assume_role(self):
env_vars = {}
if os.environ.get("AWS_ASSUME_ROLE_ARN"):
env_vars["AWS_ASSUME_ROLE_ARN"] = os.environ.get("AWS_ASSUME_ROLE_ARN")
env_vars["AWS_ASSUME_ROLE_SESSION_NAME"] = os.environ.get(
"AWS_ASSUME_ROLE_SESSION_NAME"
)
env_vars["ENV"] = os.environ.get("ENV") or "main"
def process_records(records):
print(records)
# start Kinesis client
stream_name = f"test-foobar-{short_uid()}"
kinesis_connector.listen_to_kinesis(
stream_name=stream_name,
listener_func=process_records,
env_vars=env_vars,
kcl_log_level=logging.INFO,
wait_until_started=True,
)
def test_run_kcl(self):
result = []
def process_records(records):
result.extend(records)
# start Kinesis client
stream_name = f"test-foobar-{short_uid()}"
aws_stack.create_kinesis_stream(stream_name, delete=True)
kinesis_connector.listen_to_kinesis(
stream_name=stream_name,
listener_func=process_records,
kcl_log_level=logging.INFO,
wait_until_started=True,
)
kinesis = aws_stack.create_external_boto_client("kinesis")
stream_summary = kinesis.describe_stream_summary(
StreamName=stream_name)
self.assertEqual(
1, stream_summary["StreamDescriptionSummary"]["OpenShardCount"])
num_events_kinesis = 10
kinesis.put_records(
Records=[{
"Data": "{}",
"PartitionKey": "test_%s" % i
} for i in range(0, num_events_kinesis)],
StreamName=stream_name,
)
def check_events():
self.assertEqual(num_events_kinesis, len(result))
retry(check_events, retries=4, sleep=2)
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# put items to table
num_events_ddb = 10
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events_ddb - 3):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}}
]})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
if len(EVENTS) != num_events:
LOGGER.warning('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == 10
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = '-kclapp2'
table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_service('dynamodb', client=True)
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27,
delete=True)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId0'
},
'data': {
'S': 'foobar123'
}
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId1'
},
'data': {
'S': 'foobar123'
}
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId2'
},
'data': {
'S': 'foobar123'
}
}
}
}]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId3'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId4'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId5'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId0'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId1'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId2'
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId6'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId7'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId8'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId3'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId4'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId5'
}
}
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId6'
},
'data': {
'S': 'foobar123_updated1'
}
}
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId7'
},
'data': {
'S': 'foobar123_updated1'
}
}
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId6'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId7'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId8'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
])
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
LOGGER.warning(
('DynamoDB updates retrieved (actual/expected): %s/%s') %
(len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [
json.loads(base64.b64decode(e['data'])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
removes = [
e for e in event_items if e.get('__action_type') == 'REMOVE'
]
self.assertEqual(len(inserts), num_insert)
self.assertEqual(len(modifies), num_modify)
self.assertEqual(len(removes), num_delete)
for i, event in enumerate(inserts):
self.assertNotIn('old_image', event)
self.assertEqual(inserts[i]['new_image'], {
'id': 'testId%d' % i,
'data': 'foobar123'
})
self.assertEqual(modifies[0]['old_image'], {
'id': 'testId6',
'data': 'foobar123'
})
self.assertEqual(modifies[0]['new_image'], {
'id': 'testId6',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[1]['old_image'], {
'id': 'testId7',
'data': 'foobar123'
})
self.assertEqual(modifies[1]['new_image'], {
'id': 'testId7',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[2]['old_image'], {
'id': 'testId6',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[2]['new_image'], {
'id': 'testId6',
'data': 'foobar123_updated2'
})
self.assertEqual(modifies[3]['old_image'], {
'id': 'testId7',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[3]['new_image'], {
'id': 'testId7',
'data': 'foobar123_updated2'
})
self.assertEqual(modifies[4]['old_image'], {
'id': 'testId8',
'data': 'foobar123'
})
self.assertEqual(modifies[4]['new_image'], {
'id': 'testId8',
'data': 'foobar123_updated2'
})
for i, event in enumerate(removes):
self.assertEqual(event['old_image'], {
'id': 'testId%d' % i,
'data': 'foobar123'
})
self.assertNotIn('new_image', event)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
ddb_lease_table_suffix = '-kclapp'
table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27,
delete=True)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_QUEUE,
zip_file=zip_file,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ✓'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 £'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ¢'
}
}
}]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
'S': 'testId%s' % i
}},
AttributeUpdates={
'data': {
'Action': 'PUT',
'Value': {
'S': 'foobar123_updated'
}
}
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
self.assertEqual(len(latest), 10)
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'],
MessageBody=str(i))
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_events_kinesis
def check_events():
if len(events) != num_events:
LOGGER.warning((
'DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s'
) % (len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [
json.loads(base64.b64decode(e['data'])) for e in events
]
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
self.assertEqual(len(inserts), num_put_new_items + num_batch_items)
self.assertEqual(len(modifies),
num_put_existing_items + num_updates_ddb)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# check cloudwatch notifications
num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM)
# TODO: It seems that CloudWatch is currently reporting an incorrect number of
# invocations, namely the sum over *all* lambdas, not the single one we're asking for.
# Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single
# Lambda invocation may happen with a set of Kinesis records, hence we cannot simply
# add num_events_ddb to num_events_lambda above!
# self.assertEqual(num_invocations, 2 + num_events_lambda)
self.assertGreater(num_invocations, num_events_sns + num_events_sqs)
num_error_invocations = get_lambda_invocations_count(
TEST_LAMBDA_NAME_STREAM, 'Errors')
self.assertEqual(num_error_invocations, 1)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM)
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = "-kclapp2"
table_name = TEST_TABLE_NAME + "lsbat" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
dynamodb = aws_stack.create_external_boto_client("dynamodb",
client=True)
dynamodb_service = aws_stack.create_external_boto_client("dynamodb")
dynamodbstreams = aws_stack.create_external_boto_client(
"dynamodbstreams")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
func_name=lambda_ddb_name,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId0"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId1"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId2"
},
"data": {
"S": "foobar123"
},
}
}
},
]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId3"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId4"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId5"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId0"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId1"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId2"
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId8"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId3"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId4"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId5"
}
},
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId6"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId7"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId8"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
])
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
msg = "DynamoDB updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
event_items = [
json.loads(base64.b64decode(e["data"])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get("__action_type") == "INSERT"
]
modifies = [
e for e in event_items if e.get("__action_type") == "MODIFY"
]
removes = [
e for e in event_items if e.get("__action_type") == "REMOVE"
]
self.assertEqual(num_insert, len(inserts))
self.assertEqual(num_modify, len(modifies))
self.assertEqual(num_delete, len(removes))
# assert that all inserts were received
for i, event in enumerate(inserts):
self.assertNotIn("old_image", event)
item_id = "testId%d" % i
matching = [
i for i in inserts if i["new_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["new_image"])
# assert that all updates were received
def assert_updates(expected_updates, modifies):
def found(update):
for modif in modifies:
if modif["old_image"]["id"] == update["id"]:
self.assertEqual(
modif["old_image"],
{
"id": update["id"],
"data": update["old"]
},
)
self.assertEqual(
modif["new_image"],
{
"id": update["id"],
"data": update["new"]
},
)
return True
for update in expected_updates:
self.assertTrue(found(update))
updates1 = [
{
"id": "testId6",
"old": "foobar123",
"new": "foobar123_updated1"
},
{
"id": "testId7",
"old": "foobar123",
"new": "foobar123_updated1"
},
]
updates2 = [
{
"id": "testId6",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId7",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId8",
"old": "foobar123",
"new": "foobar123_updated2"
},
]
assert_updates(updates1, modifies[:2])
assert_updates(updates2, modifies[2:])
# assert that all removes were received
for i, event in enumerate(removes):
self.assertNotIn("new_image", event)
item_id = "testId%d" % i
matching = [
i for i in removes if i["old_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["old_image"])
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=4)
# clean up
testutil.delete_lambda_function(lambda_ddb_name)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
def create_kinesis_stream(name, delete=False):
stream = aws_stack.create_kinesis_stream(name, delete=delete)
stream.wait_for()
ddb_lease_table_suffix = "-kclapp"
table_name = TEST_TABLE_NAME + "klsdss" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_stream_name = "lambda-stream-%s" % short_uid()
lambda_queue_name = "lambda-queue-%s" % short_uid()
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
queue_name = "queue-%s" % short_uid()
dynamodb = aws_stack.connect_to_resource("dynamodb")
dynamodb_service = aws_stack.create_external_boto_client("dynamodb")
dynamodbstreams = aws_stack.create_external_boto_client(
"dynamodbstreams")
kinesis = aws_stack.create_external_boto_client("kinesis")
sns = aws_stack.create_external_boto_client("sns")
sqs = aws_stack.create_external_boto_client("sqs")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
create_kinesis_stream(stream_name, delete=True)
create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
records = [
json.loads(base64.b64decode(r["data"])) if r.get("data") else r
for r in records
]
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS)
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# make sure we cannot create Lambda with same name twice
with self.assertRaises(Exception):
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)["StreamDescription"]["StreamARN"]
testutil.create_lambda_function(
func_name=lambda_stream_name,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(queue_name)
testutil.create_lambda_function(
func_name=lambda_queue_name,
zip_file=zip_file,
event_source_arn=sqs_queue_info["QueueArn"],
)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = (num_events_ddb - num_put_new_items -
num_put_existing_items - num_batch_items)
LOGGER.info("Putting %s items to table...", num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: f"testId{i}",
"data": "foobar123"
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: f"testId{i}",
"data": "foobar_put_existing"
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobaz123 ✓"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobaz123 £"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobaz123 ¢"
}
}
},
]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
"S": f"testId{i}"
}},
AttributeUpdates={
"data": {
"Action": "PUT",
"Value": {
"S": "foo_updated"
}
}
},
)
# put items to stream
num_events_kinesis = 1
num_kinesis_records = 10
LOGGER.info("Putting %s records in %s event to stream...",
num_kinesis_records, num_events_kinesis)
kinesis.put_records(
Records=[{
"Data": "{}",
"PartitionKey": f"testId{i}"
} for i in range(0, num_kinesis_records)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# put 1 item to stream that will trigger an error in the Lambda
num_events_kinesis_err = 1
for i in range(num_events_kinesis_err):
kinesis.put_record(
Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey="testIdError",
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response["TopicArn"],
Protocol="lambda",
Endpoint=aws_stack.lambda_function_arn(lambda_stream_name),
)
for i in range(num_events_sns):
sns.publish(
TopicArn=response["TopicArn"],
Subject="test_subject",
Message=f"test message {i}",
)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id="shardId-000000000000",
count=10)
self.assertEqual(10, len(latest))
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info["QueueUrl"],
MessageBody=str(i))
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_kinesis_records
def check_events():
if len(events) != num_events:
msg = "DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [e for e in events if e.get("__action_type") == "INSERT"]
modifies = [
e for e in events if e.get("__action_type") == "MODIFY"
]
self.assertEqual(num_put_new_items + num_batch_items, len(inserts))
self.assertEqual(num_put_existing_items + num_updates_ddb,
len(modifies))
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=15, sleep=2)
# check cloudwatch notifications
def check_cw_invocations():
num_invocations = get_lambda_invocations_count(lambda_stream_name)
expected_invocation_count = num_events_kinesis + num_events_kinesis_err + num_events_sns
self.assertEqual(expected_invocation_count, num_invocations)
num_error_invocations = get_lambda_invocations_count(
lambda_stream_name, "Errors")
self.assertEqual(num_events_kinesis_err, num_error_invocations)
# Lambda invocations are running asynchronously, hence sleep some time here to wait for results
retry(check_cw_invocations, retries=7, sleep=2)
# clean up
testutil.delete_lambda_function(lambda_stream_name)
testutil.delete_lambda_function(lambda_ddb_name)
testutil.delete_lambda_function(lambda_queue_name)
sqs.delete_queue(QueueUrl=sqs_queue_info["QueueUrl"])
def start_test(env=ENV_DEV):
try:
# setup environment
if env == ENV_DEV:
infra.start_infra(async=True)
time.sleep(6)
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating stream...')
aws_stack.create_kinesis_stream(TEST_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
env=env, stream_view_type='NEW_AND_OLD_IMAGES')
# list streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
event_source_arn = stream['StreamArn']
assert event_source_arn
# deploy test lambda
script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py'))
zip_file = create_lambda_archive(script, get_content=True)
create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=event_source_arn)
# put items to table
num_events = 10
print('Putting %s items to table...' % num_events)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events):
table.put_item(Item={
PARTITION_KEY: 'testId123',
'data': 'foobar123'
})
print("Waiting some time before finishing test.")
time.sleep(10)
print('DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' % (len(EVENTS), num_events))
if len(EVENTS) != num_events:
print('ERROR receiving DynamoDB updates. Running processes:')
print(run("ps aux | grep 'python\|java\|node'"))
assert len(EVENTS) == num_events
print("Test finished successfully")
cleanup(env=env)
except KeyboardInterrupt, e:
infra.KILLED = True
def test_kinesis_lambda_ddb_streams():
env = ENV_DEV
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodb_service = aws_stack.connect_to_service('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
env=env,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda (Python) connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(TEST_LAMBDA_PYTHON,
get_content=True,
libs=['localstack'],
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda (Python) connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM_PY,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda (Java) connected to Kinesis Stream
zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True)
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_STREAM_JAVA,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='com.atlassian.localstack.sample.KinesisHandler')
if use_docker():
# deploy test lambda (Node.js) connected to Kinesis Stream
zip_file = testutil.create_lambda_archive(
TEST_LAMBDA_NODEJS,
get_content=True,
runtime=LAMBDA_RUNTIME_NODEJS)
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_STREAM_JS,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_NODEJS)
# put items to table
num_events_ddb = 10
print('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events_ddb):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# put items to stream
num_events_kinesis = 10
print('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
assert len(latest) == 10
print("Waiting some time before finishing test.")
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis
print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' %
(len(EVENTS), num_events))
assert len(EVENTS) == num_events
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
TEST_TABLE_NAME: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ✓'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 £'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ¢'
}
}
}]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {
'S': 'testId%s' % i
}},
AttributeUpdates={
'data': {
'Action': 'PUT',
'Value': {
'S': 'foobar123_updated'
}
}
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(
('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
def test_kinesis_lambda_ddb_streams():
env = ENV_DEV
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating test streams...')
aws_stack.create_kinesis_stream(TEST_STREAM_NAME)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
env=env, stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py'))
zip_file = testutil.create_lambda_archive(script, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME,
zip_file=zip_file, event_source_arn=ddb_event_source_arn)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_SOURCE_STREAM_NAME,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn)
# put items to table
num_events_ddb = 10
print('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events_ddb):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# put items to stream
num_events_kinesis = 10
print('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
print("Waiting some time before finishing test.")
time.sleep(4)
num_events = num_events_ddb + num_events_kinesis
print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events))
if len(EVENTS) != num_events:
print('ERROR receiving DynamoDB updates.')
assert len(EVENTS) == num_events
def create_delivery_stream(
self,
context: RequestContext,
delivery_stream_name: DeliveryStreamName,
delivery_stream_type: DeliveryStreamType = DeliveryStreamType.
DirectPut,
kinesis_stream_source_configuration:
KinesisStreamSourceConfiguration = None,
delivery_stream_encryption_configuration_input:
DeliveryStreamEncryptionConfigurationInput = None,
s3_destination_configuration: S3DestinationConfiguration = None,
extended_s3_destination_configuration:
ExtendedS3DestinationConfiguration = None,
redshift_destination_configuration:
RedshiftDestinationConfiguration = None,
elasticsearch_destination_configuration:
ElasticsearchDestinationConfiguration = None,
amazonopensearchservice_destination_configuration:
AmazonopensearchserviceDestinationConfiguration = None,
splunk_destination_configuration: SplunkDestinationConfiguration = None,
http_endpoint_destination_configuration:
HttpEndpointDestinationConfiguration = None,
tags: TagDeliveryStreamInputTagList = None,
) -> CreateDeliveryStreamOutput:
region = FirehoseBackend.get()
destinations: DestinationDescriptionList = []
if elasticsearch_destination_configuration:
destinations.append(
DestinationDescription(
DestinationId=short_uid(),
ElasticsearchDestinationDescription=
convert_es_config_to_desc(
elasticsearch_destination_configuration),
))
if amazonopensearchservice_destination_configuration:
destinations.append(
DestinationDescription(
DestinationId=short_uid(),
AmazonopensearchserviceDestinationDescription=
convert_opensearch_config_to_desc(
amazonopensearchservice_destination_configuration),
))
if s3_destination_configuration or extended_s3_destination_configuration:
destinations.append(
DestinationDescription(
DestinationId=short_uid(),
S3DestinationDescription=convert_s3_config_to_desc(
s3_destination_configuration),
ExtendedS3DestinationDescription=
convert_extended_s3_config_to_desc(
extended_s3_destination_configuration),
))
if http_endpoint_destination_configuration:
destinations.append(
DestinationDescription(
DestinationId=short_uid(),
HttpEndpointDestinationDescription=
convert_http_config_to_desc(
http_endpoint_destination_configuration),
))
if splunk_destination_configuration:
LOG.warning(
"Delivery stream contains a splunk destination (which is currently not supported)."
)
if redshift_destination_configuration:
LOG.warning(
"Delivery stream contains a redshift destination (which is currently not supported)."
)
stream = DeliveryStreamDescription(
DeliveryStreamName=delivery_stream_name,
DeliveryStreamARN=firehose_stream_arn(
stream_name=delivery_stream_name,
account_id=context.account_id,
region_name=context.region,
),
DeliveryStreamStatus=DeliveryStreamStatus.ACTIVE,
DeliveryStreamType=delivery_stream_type,
HasMoreDestinations=False,
VersionId="1",
CreateTimestamp=datetime.now(),
LastUpdateTimestamp=datetime.now(),
Destinations=destinations,
Source=convert_source_config_to_desc(
kinesis_stream_source_configuration),
)
FirehoseBackend.TAGS.tag_resource(stream["DeliveryStreamARN"], tags)
region.delivery_streams[delivery_stream_name] = stream
# record event
event_publisher.fire_event(
event_publisher.EVENT_FIREHOSE_CREATE_STREAM,
payload={"n": event_publisher.get_hash(delivery_stream_name)},
)
if delivery_stream_type == DeliveryStreamType.KinesisStreamAsSource:
if not kinesis_stream_source_configuration:
raise InvalidArgumentException(
"Missing delivery stream configuration")
kinesis_stream_name = kinesis_stream_source_configuration[
"KinesisStreamARN"].split("/")[1]
kinesis_connector.listen_to_kinesis(
stream_name=kinesis_stream_name,
fh_d_stream=delivery_stream_name,
listener_func=self._process_records,
wait_until_started=True,
ddb_lease_table_suffix="-firehose",
)
return CreateDeliveryStreamOutput(
DeliveryStreamARN=stream["DeliveryStreamARN"])
def create_stream(
stream_name: str,
delivery_stream_type: str = "DirectPut",
delivery_stream_type_configuration: Dict = None,
s3_destination: Dict = None,
elasticsearch_destination: Dict = None,
http_destination: Dict = None,
tags: Dict[str, str] = None,
):
"""Create a firehose stream with destination configurations. In case 'KinesisStreamAsSource' is set,
creates a listener to process records from the underlying kinesis stream."""
region = FirehoseBackend.get()
tags = tags or {}
stream = {
"DeliveryStreamType": delivery_stream_type,
"KinesisStreamSourceConfiguration": delivery_stream_type_configuration,
"HasMoreDestinations": False,
"VersionId": "1",
"CreateTimestamp": time.time(),
"DeliveryStreamARN": firehose_stream_arn(stream_name),
"DeliveryStreamStatus": "ACTIVE",
"DeliveryStreamName": stream_name,
"Destinations": [],
"Tags": tags,
}
region.delivery_streams[stream_name] = stream
if elasticsearch_destination:
update_destination(
stream_name=stream_name,
destination_id=short_uid(),
elasticsearch_update=elasticsearch_destination,
)
if s3_destination:
update_destination(
stream_name=stream_name,
destination_id=short_uid(),
s3_update=s3_destination,
)
if http_destination:
update_destination(
stream_name=stream_name,
destination_id=short_uid(),
http_update=http_destination,
)
# record event
event_publisher.fire_event(
event_publisher.EVENT_FIREHOSE_CREATE_STREAM,
payload={"n": event_publisher.get_hash(stream_name)},
)
if delivery_stream_type == "KinesisStreamAsSource":
kinesis_stream_name = delivery_stream_type_configuration.get(
"KinesisStreamARN").split("/")[1]
kinesis_connector.listen_to_kinesis(
stream_name=kinesis_stream_name,
fh_d_stream=stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix="-firehose",
)
return stream
def test_kinesis_lambda_parallelism(self, lambda_client, kinesis_client):
function_name = f"lambda_func-{short_uid()}"
stream_name = f"test-foobar-{short_uid()}"
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PARALLEL_FILE,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
arn = aws_stack.kinesis_stream_arn(stream_name, account_id="000000000000")
lambda_client.create_event_source_mapping(EventSourceArn=arn, FunctionName=function_name)
def process_records(record):
assert record
aws_stack.create_kinesis_stream(stream_name, delete=True)
kinesis_connector.listen_to_kinesis(
stream_name=stream_name,
listener_func=process_records,
wait_until_started=True,
)
kinesis = aws_stack.create_external_boto_client("kinesis")
stream_summary = kinesis.describe_stream_summary(StreamName=stream_name)
assert 1 == stream_summary["StreamDescriptionSummary"]["OpenShardCount"]
num_events_kinesis = 10
# assure async call
start = time.perf_counter()
kinesis.put_records(
Records=[
{"Data": '{"batch": 0}', "PartitionKey": f"test_{i}"}
for i in range(0, num_events_kinesis)
],
StreamName=stream_name,
)
assert (time.perf_counter() - start) < 1 # this should not take more than a second
kinesis.put_records(
Records=[
{"Data": '{"batch": 1}', "PartitionKey": f"test_{i}"}
for i in range(0, num_events_kinesis)
],
StreamName=stream_name,
)
def get_events():
events = get_lambda_log_events(function_name, regex_filter=r"event.*Records")
assert len(events) == 2
return events
events = retry(get_events, retries=5)
def assertEvent(event, batch_no):
assert 10 == len(event["event"]["Records"])
assert "eventID" in event["event"]["Records"][0]
assert "eventSourceARN" in event["event"]["Records"][0]
assert "eventSource" in event["event"]["Records"][0]
assert "eventVersion" in event["event"]["Records"][0]
assert "eventName" in event["event"]["Records"][0]
assert "invokeIdentityArn" in event["event"]["Records"][0]
assert "awsRegion" in event["event"]["Records"][0]
assert "kinesis" in event["event"]["Records"][0]
assert {"batch": batch_no} == json.loads(
base64.b64decode(event["event"]["Records"][0]["kinesis"]["data"]).decode(
config.DEFAULT_ENCODING
)
)
assertEvent(events[0], 0)
assertEvent(events[1], 1)
assert (events[1]["executionStart"] - events[0]["executionStart"]) > 5
# cleanup
lambda_client.delete_function(FunctionName=function_name)
kinesis_client.delete_stream(StreamName=stream_name)
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
]})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {'S': 'testId%s' % i}},
AttributeUpdates={'data': {
'Action': 'PUT',
'Value': {'S': 'foobar123_updated'}
}})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
