def handle_request(s_src, thread):
s_dst = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s_dst.connect(ip_to_tuple(dst))
sockets = [s_src, s_dst]
try:
while thread.running:
s_read, _, _ = select.select(sockets, [], [])
for s in s_read:
data = s.recv(BUFFER_SIZE)
if data in [b"", "", None]:
return
if s == s_src:
forward, response = data, None
if handler:
forward, response = handler(data)
if forward is not None:
s_dst.sendall(forward)
elif response is not None:
s_src.sendall(response)
return
elif s == s_dst:
s_src.sendall(data)
finally:
run_safe(s_src.close)
run_safe(s_dst.close)
def remove_nones(o, **kwargs):
if isinstance(o, dict):
for k, v in dict(o).items():
if v is None:
o.pop(k)
if isinstance(o, list):
common.run_safe(o.remove, None)
common.run_safe(o.remove, PLACEHOLDER_AWS_NO_VALUE)
return o
def create_kinesis_stream(stream_name, shards=1, env=None, delete=False):
env = get_environment(env)
stream = KinesisStream(id=stream_name, num_shards=shards)
conn = connect_to_service("kinesis", env=env)
stream.connect(conn)
if delete:
run_safe(lambda: stream.destroy(), print_error=False)
stream.create()
# Note: Returning the stream without awaiting its creation (via wait_for()) to avoid API call timeouts/retries.
return stream
def create_kinesis_stream(stream_name, shards=1, env=None, delete=False):
env = get_environment(env)
# stream
stream = KinesisStream(id=stream_name, num_shards=shards)
conn = connect_to_service('kinesis', env=env)
stream.connect(conn)
if delete:
run_safe(lambda: stream.destroy(), print_error=False)
stream.create()
stream.wait_for()
return stream
def create_kinesis_stream(stream_name, shards=1, env=None, delete=False):
env = get_environment(env)
# stream
stream = KinesisStream(id=stream_name, num_shards=shards)
conn = connect_to_service('kinesis', env=env)
stream.connect(conn)
if delete:
run_safe(lambda: stream.destroy(), print_error=False)
stream.create()
stream.wait_for()
return stream
def get_template_body(req_data):
body = req_data.get("TemplateBody")
if body:
return body
url = req_data.get("TemplateURL")
if url:
response = run_safe(lambda: safe_requests.get(url, verify=False))
# check error codes, and code 301 - fixes https://github.com/localstack/localstack/issues/1884
status_code = 0 if response is None else response.status_code
if response is None or status_code == 301 or status_code >= 400:
# check if this is an S3 URL, then get the file directly from there
url = convert_s3_to_local_url(url)
if is_local_service_url(url):
parsed_path = urlparse.urlparse(url).path.lstrip("/")
parts = parsed_path.partition("/")
client = aws_stack.connect_to_service("s3")
LOG.debug(
"Download CloudFormation template content from local S3: %s - %s"
% (parts[0], parts[2]))
result = client.get_object(Bucket=parts[0], Key=parts[2])
body = to_str(result["Body"].read())
return body
raise Exception(
"Unable to fetch template body (code %s) from URL %s" %
(status_code, url))
return response.content
raise Exception("Unable to get template body from input: %s" % req_data)
def test_adding_fallback_function_name_in_headers(self):
lambda_client = aws_stack.create_external_boto_client("lambda")
ddb_client = aws_stack.create_external_boto_client("dynamodb")
db_table = "lambda-records"
config.LAMBDA_FALLBACK_URL = "dynamodb://%s" % db_table
lambda_client.invoke(
FunctionName="non-existing-lambda",
Payload=b"{}",
InvocationType="RequestResponse",
)
result = run_safe(ddb_client.scan, TableName=db_table)
self.assertEqual("non-existing-lambda", result["Items"][0]["function_name"]["S"])
def test_adding_fallback_function_name_in_headers(self):
lambda_client = aws_stack.connect_to_service('lambda')
ddb_client = aws_stack.connect_to_service('dynamodb')
db_table = 'lambda-records'
config.LAMBDA_FALLBACK_URL = 'dynamodb://%s' % db_table
lambda_client.invoke(FunctionName='non-existing-lambda',
Payload=b'{}',
InvocationType='RequestResponse')
result = run_safe(ddb_client.scan, TableName=db_table)
self.assertEqual(result['Items'][0]['function_name']['S'],
'non-existing-lambda')
def get_template_body(req_data):
body = req_data.get('TemplateBody')
if body:
return body
url = req_data.get('TemplateURL')
if url:
response = run_safe(lambda: safe_requests.get(url, verify=False))
if not response or response.status_code >= 400:
# check if this is an S3 URL, then get the file directly from there
if '://localhost' in url or re.match(r'.*s3(\-website)?\.([^\.]+\.)?amazonaws.com.*', url):
parsed_path = urlparse.urlparse(url).path.lstrip('/')
parts = parsed_path.partition('/')
client = aws_stack.connect_to_service('s3')
result = client.get_object(Bucket=parts[0], Key=parts[2])
body = to_str(result['Body'].read())
return body
raise Exception('Unable to fetch template body (code %s) from URL %s' % (response.status_code, url))
return response.content
raise Exception('Unable to get template body from input: %s' % req_data)
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
TEST_TABLE_NAME: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ✓'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 £'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ¢'
}
}
}]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {
'S': 'testId%s' % i
}},
AttributeUpdates={
'data': {
'Action': 'PUT',
'Value': {
'S': 'foobar123_updated'
}
}
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(
('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = '-kclapp2'
table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_service('dynamodb', client=True)
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27,
delete=True)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId0'
},
'data': {
'S': 'foobar123'
}
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId1'
},
'data': {
'S': 'foobar123'
}
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId2'
},
'data': {
'S': 'foobar123'
}
}
}
}]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId3'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId4'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId5'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId0'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId1'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId2'
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId6'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId7'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId8'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId3'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId4'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId5'
}
}
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId6'
},
'data': {
'S': 'foobar123_updated1'
}
}
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId7'
},
'data': {
'S': 'foobar123_updated1'
}
}
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId6'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId7'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId8'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
])
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
LOGGER.warning(
('DynamoDB updates retrieved (actual/expected): %s/%s') %
(len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [
json.loads(base64.b64decode(e['data'])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
removes = [
e for e in event_items if e.get('__action_type') == 'REMOVE'
]
self.assertEqual(len(inserts), num_insert)
self.assertEqual(len(modifies), num_modify)
self.assertEqual(len(removes), num_delete)
for i, event in enumerate(inserts):
self.assertNotIn('old_image', event)
self.assertEqual(inserts[i]['new_image'], {
'id': 'testId%d' % i,
'data': 'foobar123'
})
self.assertEqual(modifies[0]['old_image'], {
'id': 'testId6',
'data': 'foobar123'
})
self.assertEqual(modifies[0]['new_image'], {
'id': 'testId6',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[1]['old_image'], {
'id': 'testId7',
'data': 'foobar123'
})
self.assertEqual(modifies[1]['new_image'], {
'id': 'testId7',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[2]['old_image'], {
'id': 'testId6',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[2]['new_image'], {
'id': 'testId6',
'data': 'foobar123_updated2'
})
self.assertEqual(modifies[3]['old_image'], {
'id': 'testId7',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[3]['new_image'], {
'id': 'testId7',
'data': 'foobar123_updated2'
})
self.assertEqual(modifies[4]['old_image'], {
'id': 'testId8',
'data': 'foobar123'
})
self.assertEqual(modifies[4]['new_image'], {
'id': 'testId8',
'data': 'foobar123_updated2'
})
for i, event in enumerate(removes):
self.assertEqual(event['old_image'], {
'id': 'testId%d' % i,
'data': 'foobar123'
})
self.assertNotIn('new_image', event)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
ddb_lease_table_suffix = '-kclapp'
table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27,
delete=True)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_QUEUE,
zip_file=zip_file,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ✓'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 £'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ¢'
}
}
}]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
'S': 'testId%s' % i
}},
AttributeUpdates={
'data': {
'Action': 'PUT',
'Value': {
'S': 'foobar123_updated'
}
}
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
self.assertEqual(len(latest), 10)
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'],
MessageBody=str(i))
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_events_kinesis
def check_events():
if len(events) != num_events:
LOGGER.warning((
'DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s'
) % (len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [
json.loads(base64.b64decode(e['data'])) for e in events
]
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
self.assertEqual(len(inserts), num_put_new_items + num_batch_items)
self.assertEqual(len(modifies),
num_put_existing_items + num_updates_ddb)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# check cloudwatch notifications
num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM)
# TODO: It seems that CloudWatch is currently reporting an incorrect number of
# invocations, namely the sum over *all* lambdas, not the single one we're asking for.
# Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single
# Lambda invocation may happen with a set of Kinesis records, hence we cannot simply
# add num_events_ddb to num_events_lambda above!
# self.assertEqual(num_invocations, 2 + num_events_lambda)
self.assertGreater(num_invocations, num_events_sns + num_events_sqs)
num_error_invocations = get_lambda_invocations_count(
TEST_LAMBDA_NAME_STREAM, 'Errors')
self.assertEqual(num_error_invocations, 1)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM)
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = "-kclapp2"
table_name = TEST_TABLE_NAME + "lsbat" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
dynamodb = aws_stack.create_external_boto_client("dynamodb",
client=True)
dynamodb_service = aws_stack.create_external_boto_client("dynamodb")
dynamodbstreams = aws_stack.create_external_boto_client(
"dynamodbstreams")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
func_name=lambda_ddb_name,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId0"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId1"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId2"
},
"data": {
"S": "foobar123"
},
}
}
},
]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId3"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId4"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId5"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId0"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId1"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId2"
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId8"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId3"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId4"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId5"
}
},
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId6"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId7"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId8"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
])
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
msg = "DynamoDB updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
event_items = [
json.loads(base64.b64decode(e["data"])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get("__action_type") == "INSERT"
]
modifies = [
e for e in event_items if e.get("__action_type") == "MODIFY"
]
removes = [
e for e in event_items if e.get("__action_type") == "REMOVE"
]
self.assertEqual(num_insert, len(inserts))
self.assertEqual(num_modify, len(modifies))
self.assertEqual(num_delete, len(removes))
# assert that all inserts were received
for i, event in enumerate(inserts):
self.assertNotIn("old_image", event)
item_id = "testId%d" % i
matching = [
i for i in inserts if i["new_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["new_image"])
# assert that all updates were received
def assert_updates(expected_updates, modifies):
def found(update):
for modif in modifies:
if modif["old_image"]["id"] == update["id"]:
self.assertEqual(
modif["old_image"],
{
"id": update["id"],
"data": update["old"]
},
)
self.assertEqual(
modif["new_image"],
{
"id": update["id"],
"data": update["new"]
},
)
return True
for update in expected_updates:
self.assertTrue(found(update))
updates1 = [
{
"id": "testId6",
"old": "foobar123",
"new": "foobar123_updated1"
},
{
"id": "testId7",
"old": "foobar123",
"new": "foobar123_updated1"
},
]
updates2 = [
{
"id": "testId6",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId7",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId8",
"old": "foobar123",
"new": "foobar123_updated2"
},
]
assert_updates(updates1, modifies[:2])
assert_updates(updates2, modifies[2:])
# assert that all removes were received
for i, event in enumerate(removes):
self.assertNotIn("new_image", event)
item_id = "testId%d" % i
matching = [
i for i in removes if i["old_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["old_image"])
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=4)
# clean up
testutil.delete_lambda_function(lambda_ddb_name)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
def create_kinesis_stream(name, delete=False):
stream = aws_stack.create_kinesis_stream(name, delete=delete)
stream.wait_for()
ddb_lease_table_suffix = "-kclapp"
table_name = TEST_TABLE_NAME + "klsdss" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_stream_name = "lambda-stream-%s" % short_uid()
lambda_queue_name = "lambda-queue-%s" % short_uid()
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
queue_name = "queue-%s" % short_uid()
dynamodb = aws_stack.connect_to_resource("dynamodb")
dynamodb_service = aws_stack.create_external_boto_client("dynamodb")
dynamodbstreams = aws_stack.create_external_boto_client(
"dynamodbstreams")
kinesis = aws_stack.create_external_boto_client("kinesis")
sns = aws_stack.create_external_boto_client("sns")
sqs = aws_stack.create_external_boto_client("sqs")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
create_kinesis_stream(stream_name, delete=True)
create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
records = [
json.loads(base64.b64decode(r["data"])) if r.get("data") else r
for r in records
]
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS)
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# make sure we cannot create Lambda with same name twice
with self.assertRaises(Exception):
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)["StreamDescription"]["StreamARN"]
testutil.create_lambda_function(
func_name=lambda_stream_name,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(queue_name)
testutil.create_lambda_function(
func_name=lambda_queue_name,
zip_file=zip_file,
event_source_arn=sqs_queue_info["QueueArn"],
)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = (num_events_ddb - num_put_new_items -
num_put_existing_items - num_batch_items)
LOGGER.info("Putting %s items to table...", num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: f"testId{i}",
"data": "foobar123"
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: f"testId{i}",
"data": "foobar_put_existing"
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobaz123 ✓"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobaz123 £"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobaz123 ¢"
}
}
},
]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
"S": f"testId{i}"
}},
AttributeUpdates={
"data": {
"Action": "PUT",
"Value": {
"S": "foo_updated"
}
}
},
)
# put items to stream
num_events_kinesis = 1
num_kinesis_records = 10
LOGGER.info("Putting %s records in %s event to stream...",
num_kinesis_records, num_events_kinesis)
kinesis.put_records(
Records=[{
"Data": "{}",
"PartitionKey": f"testId{i}"
} for i in range(0, num_kinesis_records)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# put 1 item to stream that will trigger an error in the Lambda
num_events_kinesis_err = 1
for i in range(num_events_kinesis_err):
kinesis.put_record(
Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey="testIdError",
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response["TopicArn"],
Protocol="lambda",
Endpoint=aws_stack.lambda_function_arn(lambda_stream_name),
)
for i in range(num_events_sns):
sns.publish(
TopicArn=response["TopicArn"],
Subject="test_subject",
Message=f"test message {i}",
)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id="shardId-000000000000",
count=10)
self.assertEqual(10, len(latest))
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info["QueueUrl"],
MessageBody=str(i))
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_kinesis_records
def check_events():
if len(events) != num_events:
msg = "DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [e for e in events if e.get("__action_type") == "INSERT"]
modifies = [
e for e in events if e.get("__action_type") == "MODIFY"
]
self.assertEqual(num_put_new_items + num_batch_items, len(inserts))
self.assertEqual(num_put_existing_items + num_updates_ddb,
len(modifies))
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=15, sleep=2)
# check cloudwatch notifications
def check_cw_invocations():
num_invocations = get_lambda_invocations_count(lambda_stream_name)
expected_invocation_count = num_events_kinesis + num_events_kinesis_err + num_events_sns
self.assertEqual(expected_invocation_count, num_invocations)
num_error_invocations = get_lambda_invocations_count(
lambda_stream_name, "Errors")
self.assertEqual(num_events_kinesis_err, num_error_invocations)
# Lambda invocations are running asynchronously, hence sleep some time here to wait for results
retry(check_cw_invocations, retries=7, sleep=2)
# clean up
testutil.delete_lambda_function(lambda_stream_name)
testutil.delete_lambda_function(lambda_ddb_name)
testutil.delete_lambda_function(lambda_queue_name)
sqs.delete_queue(QueueUrl=sqs_queue_info["QueueUrl"])
def num_items():
return len((run_safe(ddb_client.scan, TableName=db_table) or {
'Items': []
})['Items'])
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
]})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {'S': 'testId%s' % i}},
AttributeUpdates={'data': {
'Action': 'PUT',
'Value': {'S': 'foobar123_updated'}
}})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
def num_items():
return len((run_safe(ddb_client.scan, TableName=db_table) or {"Items": []})["Items"])
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# put items to table
num_events_ddb = 10
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events_ddb - 3):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}}
]})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
if len(EVENTS) != num_events:
LOGGER.warning('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == 10
def check_item():
result = run_safe(ddb_client.scan, TableName=db_table)
self.assertEqual("non-existing-lambda",
result["Items"][0]["function_name"]["S"])
