def test_kinesis_lambda_forward_chain(): kinesis = aws_stack.connect_to_service('kinesis') s3 = aws_stack.connect_to_service('s3') aws_stack.create_kinesis_stream(TEST_CHAIN_STREAM1_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_CHAIN_STREAM2_NAME, delete=True) s3.create_bucket(Bucket=TEST_BUCKET_NAME) # deploy test lambdas connected to Kinesis streams zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_CHAIN_LAMBDA1_NAME, zip_file=zip_file, event_source_arn=get_event_source_arn(TEST_CHAIN_STREAM1_NAME), runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_CHAIN_LAMBDA2_NAME, zip_file=zip_file, event_source_arn=get_event_source_arn(TEST_CHAIN_STREAM2_NAME), runtime=LAMBDA_RUNTIME_PYTHON27) # publish test record test_data = {'test_data': 'forward_chain_data_%s' % short_uid()} data = clone(test_data) data[lambda_integration.MSG_BODY_MESSAGE_TARGET] = 'kinesis:%s' % TEST_CHAIN_STREAM2_NAME kinesis.put_record(Data=to_bytes(json.dumps(data)), PartitionKey='testId', StreamName=TEST_CHAIN_STREAM1_NAME) # check results time.sleep(5) all_objects = testutil.list_all_s3_objects() testutil.assert_objects(test_data, all_objects)
def test_kinesis_error_injection(): if not do_run(): return kinesis = aws_stack.connect_to_service('kinesis') aws_stack.create_kinesis_stream(TEST_STREAM_NAME) records = [ { 'Data': '0', 'ExplicitHashKey': '0', 'PartitionKey': '0' } ] # by default, no errors test_no_errors = kinesis.put_records(StreamName=TEST_STREAM_NAME, Records=records) assert_equal(test_no_errors['FailedRecordCount'], 0) # with a probability of 1, always throw errors config.KINESIS_ERROR_PROBABILITY = 1.0 test_all_errors = kinesis.put_records(StreamName=TEST_STREAM_NAME, Records=records) assert_equal(test_all_errors['FailedRecordCount'], 1) # reset probability to zero config.KINESIS_ERROR_PROBABILITY = 0.0
def add_dynamodb_stream(table_name, view_type='NEW_AND_OLD_IMAGES', enabled=True): if enabled: # create kinesis stream as a backend stream_name = get_kinesis_stream_name(table_name) aws_stack.create_kinesis_stream(stream_name) stream = { 'StreamArn': aws_stack.dynamodb_stream_arn(table_name=table_name), 'TableName': table_name, 'StreamLabel': 'TODO', 'StreamStatus': 'ENABLED', 'KeySchema': [], 'Shards': [] } table_arn = aws_stack.dynamodb_table_arn(table_name) DDB_STREAMS[table_arn] = stream
def test_api_gateway_kinesis_integration(): # create target Kinesis stream aws_stack.create_kinesis_stream(TEST_STREAM_KINESIS_API_GW) # create API Gateway and connect it to the target stream result = connect_api_gateway_to_kinesis('test_gateway1', TEST_STREAM_KINESIS_API_GW) # generate test data test_data = {'records': [ {'data': '{"foo": "bar1"}'}, {'data': '{"foo": "bar2"}'}, {'data': '{"foo": "bar3"}'} ]} url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'], stage_name=TEST_STAGE_NAME, path=API_PATH_DATA_INBOUND) result = requests.post(url, data=json.dumps(test_data)) result = json.loads(to_str(result.content)) assert result['FailedRecordCount'] == 0 assert len(result['Records']) == len(test_data['records'])
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # put items to table num_events_ddb = 15 num_put_items = 7 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_put_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item( RequestItems={ TEST_TABLE_NAME: [{ 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 £' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢' } } }] }) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item( TableName=TEST_TABLE_NAME, Key={PARTITION_KEY: { 'S': 'testId%s' % i }}, AttributeUpdates={ 'data': { 'Action': 'PUT', 'Value': { 'S': 'foobar123_updated' } } }) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe( TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns def check_events(): if len(EVENTS) != num_events: LOGGER.warning( ('DynamoDB and Kinesis updates retrieved ' + '(actual/expected): %s/%s') % (len(EVENTS), num_events)) assert len(EVENTS) == num_events # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=7, sleep=3) # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == num_events_ddb
def test_kinesis_lambda_ddb_streams(): env = ENV_DEV ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodb_service = aws_stack.connect_to_service('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) print("Kinesis consumer initialized.") # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda (Python) connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(TEST_LAMBDA_PYTHON, get_content=True, libs=['localstack'], runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda (Python) connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM_PY, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda (Java) connected to Kinesis Stream zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_STREAM_JAVA, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_JAVA8, handler='com.atlassian.localstack.sample.KinesisHandler') if use_docker(): # deploy test lambda (Node.js) connected to Kinesis Stream zip_file = testutil.create_lambda_archive( TEST_LAMBDA_NODEJS, get_content=True, runtime=LAMBDA_RUNTIME_NODEJS) kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_STREAM_JS, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_NODEJS) # put items to table num_events_ddb = 10 print('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events_ddb): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # put items to stream num_events_kinesis = 10 print('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 print("Waiting some time before finishing test.") time.sleep(2) num_events = num_events_ddb + num_events_kinesis print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events)) assert len(EVENTS) == num_events
def test_lambda_streams_batch_and_transactions(self): ddb_lease_table_suffix = "-kclapp2" table_name = TEST_TABLE_NAME + "lsbat" + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME lambda_ddb_name = "lambda-ddb-%s" % short_uid() dynamodb = aws_stack.create_external_boto_client("dynamodb", client=True) dynamodb_service = aws_stack.create_external_boto_client("dynamodb") dynamodbstreams = aws_stack.create_external_boto_client( "dynamodbstreams") LOGGER.info("Creating test streams...") run_safe( lambda: dynamodb_service.delete_table(TableName=stream_name + ddb_lease_table_suffix), print_error=False, ) aws_stack.create_kinesis_stream(stream_name, delete=True) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix, ) LOGGER.info("Kinesis consumer initialized.") # create table with stream forwarding config aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type="NEW_AND_OLD_IMAGES", ) # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams["Streams"]: if stream["TableName"] == table_name: ddb_event_source_arn = stream["StreamArn"] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, libs=TEST_LAMBDA_LIBS, func_name=lambda_ddb_name, event_source_arn=ddb_event_source_arn, starting_position="TRIM_HORIZON", delete=True, ) # submit a batch with writes dynamodb.batch_write_item( RequestItems={ table_name: [ { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId0" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId1" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId2" }, "data": { "S": "foobar123" }, } } }, ] }) # submit a batch with writes and deletes dynamodb.batch_write_item( RequestItems={ table_name: [ { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId3" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId4" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId5" }, "data": { "S": "foobar123" }, } } }, { "DeleteRequest": { "Key": { PARTITION_KEY: { "S": "testId0" } } } }, { "DeleteRequest": { "Key": { PARTITION_KEY: { "S": "testId1" } } } }, { "DeleteRequest": { "Key": { PARTITION_KEY: { "S": "testId2" } } } }, ] }) # submit a transaction with writes and delete dynamodb.transact_write_items(TransactItems=[ { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId6" }, "data": { "S": "foobar123" }, }, } }, { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId7" }, "data": { "S": "foobar123" }, }, } }, { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId8" }, "data": { "S": "foobar123" }, }, } }, { "Delete": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId3" } }, } }, { "Delete": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId4" } }, } }, { "Delete": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId5" } }, } }, ]) # submit a batch with a put over existing item dynamodb.transact_write_items(TransactItems=[ { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId6" }, "data": { "S": "foobar123_updated1" }, }, } }, ]) # submit a transaction with a put over existing item dynamodb.transact_write_items(TransactItems=[ { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId7" }, "data": { "S": "foobar123_updated1" }, }, } }, ]) # submit a transaction with updates dynamodb.transact_write_items(TransactItems=[ { "Update": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId6" } }, "UpdateExpression": "SET #0 = :0", "ExpressionAttributeNames": { "#0": "data" }, "ExpressionAttributeValues": { ":0": { "S": "foobar123_updated2" } }, } }, { "Update": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId7" } }, "UpdateExpression": "SET #0 = :0", "ExpressionAttributeNames": { "#0": "data" }, "ExpressionAttributeValues": { ":0": { "S": "foobar123_updated2" } }, } }, { "Update": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId8" } }, "UpdateExpression": "SET #0 = :0", "ExpressionAttributeNames": { "#0": "data" }, "ExpressionAttributeValues": { ":0": { "S": "foobar123_updated2" } }, } }, ]) LOGGER.info("Waiting some time before finishing test.") time.sleep(2) num_insert = 9 num_modify = 5 num_delete = 6 num_events = num_insert + num_modify + num_delete def check_events(): if len(events) != num_events: msg = "DynamoDB updates retrieved (actual/expected): %s/%s" % ( len(events), num_events, ) LOGGER.warning(msg) self.assertEqual(num_events, len(events)) event_items = [ json.loads(base64.b64decode(e["data"])) for e in events ] # make sure the we have the right amount of expected event types inserts = [ e for e in event_items if e.get("__action_type") == "INSERT" ] modifies = [ e for e in event_items if e.get("__action_type") == "MODIFY" ] removes = [ e for e in event_items if e.get("__action_type") == "REMOVE" ] self.assertEqual(num_insert, len(inserts)) self.assertEqual(num_modify, len(modifies)) self.assertEqual(num_delete, len(removes)) # assert that all inserts were received for i, event in enumerate(inserts): self.assertNotIn("old_image", event) item_id = "testId%d" % i matching = [ i for i in inserts if i["new_image"]["id"] == item_id ][0] self.assertEqual({ "id": item_id, "data": "foobar123" }, matching["new_image"]) # assert that all updates were received def assert_updates(expected_updates, modifies): def found(update): for modif in modifies: if modif["old_image"]["id"] == update["id"]: self.assertEqual( modif["old_image"], { "id": update["id"], "data": update["old"] }, ) self.assertEqual( modif["new_image"], { "id": update["id"], "data": update["new"] }, ) return True for update in expected_updates: self.assertTrue(found(update)) updates1 = [ { "id": "testId6", "old": "foobar123", "new": "foobar123_updated1" }, { "id": "testId7", "old": "foobar123", "new": "foobar123_updated1" }, ] updates2 = [ { "id": "testId6", "old": "foobar123_updated1", "new": "foobar123_updated2", }, { "id": "testId7", "old": "foobar123_updated1", "new": "foobar123_updated2", }, { "id": "testId8", "old": "foobar123", "new": "foobar123_updated2" }, ] assert_updates(updates1, modifies[:2]) assert_updates(updates2, modifies[2:]) # assert that all removes were received for i, event in enumerate(removes): self.assertNotIn("new_image", event) item_id = "testId%d" % i matching = [ i for i in removes if i["old_image"]["id"] == item_id ][0] self.assertEqual({ "id": item_id, "data": "foobar123" }, matching["old_image"]) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=30, sleep=4) # clean up testutil.delete_lambda_function(lambda_ddb_name)
def test_kinesis_lambda_sns_ddb_sqs_streams(self): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') sqs = aws_stack.connect_to_service('sqs') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config aws_stack.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to SQS queue sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_QUEUE, zip_file=zip_file, event_source_arn=sqs_queue_info['QueueArn'], runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item( RequestItems={ TEST_TABLE_NAME: [{ 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 £' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢' } } }] }) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item( TableName=TEST_TABLE_NAME, Key={PARTITION_KEY: { 'S': 'testId%s' % i }}, AttributeUpdates={ 'data': { 'Action': 'PUT', 'Value': { 'S': 'foobar123_updated' } } }) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test messages num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe( TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) self.assertEqual(len(latest), 10) # send messages to SQS queue num_events_sqs = 4 for i in range(num_events_sqs): sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'], MessageBody=str(i)) LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs num_events = num_events_lambda + num_events_kinesis def check_events(): if len(EVENTS) != num_events: LOGGER.warning(( 'DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' ) % (len(EVENTS), num_events)) self.assertEqual(len(EVENTS), num_events) event_items = [ json.loads(base64.b64decode(e['data'])) for e in EVENTS ] # make sure the we have the right amount of INSERT/MODIFY event types inserts = [ e for e in event_items if e.get('__action_type') == 'INSERT' ] modifies = [ e for e in event_items if e.get('__action_type') == 'MODIFY' ] self.assertEqual(len(inserts), num_put_new_items + num_batch_items) self.assertEqual(len(modifies), num_put_existing_items + num_updates_ddb) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=3) # check cloudwatch notifications num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM) # TODO: It seems that CloudWatch is currently reporting an incorrect number of # invocations, namely the sum over *all* lambdas, not the single one we're asking for. # Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single # Lambda invocation may happen with a set of Kinesis records, hence we cannot simply # add num_events_ddb to num_events_lambda above! # self.assertEqual(num_invocations, 2 + num_events_lambda) self.assertGreater(num_invocations, num_events_sns + num_events_sqs) num_error_invocations = get_lambda_invocations_count( TEST_LAMBDA_NAME_STREAM, 'Errors') self.assertEqual(num_error_invocations, 1)
def create_kinesis_stream(name, delete=False): stream = aws_stack.create_kinesis_stream(name, delete=delete) stream.wait_for()
def test_kinesis_lambda_ddb_streams(): env = ENV_DEV dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating test streams...') aws_stack.create_kinesis_stream(TEST_STREAM_NAME) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True) print("Kinesis consumer initialized.") # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py')) zip_file = testutil.create_lambda_archive(script, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=ddb_event_source_arn) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_SOURCE_STREAM_NAME, zip_file=zip_file, event_source_arn=kinesis_event_source_arn) # put items to table num_events_ddb = 10 print('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events_ddb): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # put items to stream num_events_kinesis = 10 print('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) print("Waiting some time before finishing test.") time.sleep(4) num_events = num_events_ddb + num_events_kinesis print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events)) if len(EVENTS) != num_events: print('ERROR receiving DynamoDB updates.') assert len(EVENTS) == num_events
def test_lambda_streams_batch_and_transactions(self): ddb_lease_table_suffix = '-kclapp2' table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME lambda_ddb_name = 'lambda-ddb-%s' % short_uid() dynamodb = aws_stack.connect_to_service('dynamodb', client=True) dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=stream_name + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(stream_name, delete=True) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix ) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES' ) # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == table_name: ddb_event_source_arn = stream['StreamArn'] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, libs=TEST_LAMBDA_LIBS, func_name=lambda_ddb_name, event_source_arn=ddb_event_source_arn, delete=True) # submit a batch with writes dynamodb.batch_write_item(RequestItems={table_name: [ {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId0'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId1'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId2'}, 'data': {'S': 'foobar123'}}}} ]}) # submit a batch with writes and deletes dynamodb.batch_write_item(RequestItems={table_name: [ {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId3'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId4'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId5'}, 'data': {'S': 'foobar123'}}}}, {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId0'}}}}, {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId1'}}}}, {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId2'}}}}, ]}) # submit a transaction with writes and delete dynamodb.transact_write_items(TransactItems=[ {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123'}}}}, {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123'}}}}, {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId8'}, 'data': {'S': 'foobar123'}}}}, {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId3'}}}}, {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId4'}}}}, {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId5'}}}}, ]) # submit a batch with a put over existing item dynamodb.transact_write_items(TransactItems=[ { 'Put': { 'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123_updated1'}} } }, ]) # submit a transaction with a put over existing item dynamodb.transact_write_items(TransactItems=[ {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123_updated1'}}}}, ]) # submit a transaction with updates dynamodb.transact_write_items(TransactItems=[ {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId6'}}, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': {'#0': 'data'}, 'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}}, {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId7'}}, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': {'#0': 'data'}, 'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}}, {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId8'}}, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': {'#0': 'data'}, 'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}}, ]) LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_insert = 9 num_modify = 5 num_delete = 6 num_events = num_insert + num_modify + num_delete def check_events(): if len(events) != num_events: msg = 'DynamoDB updates retrieved (actual/expected): %s/%s' % (len(events), num_events) LOGGER.warning(msg) self.assertEqual(num_events, len(events)) event_items = [json.loads(base64.b64decode(e['data'])) for e in events] # make sure the we have the right amount of expected event types inserts = [e for e in event_items if e.get('__action_type') == 'INSERT'] modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY'] removes = [e for e in event_items if e.get('__action_type') == 'REMOVE'] self.assertEqual(num_insert, len(inserts)) self.assertEqual(num_modify, len(modifies)) self.assertEqual(num_delete, len(removes)) # assert that all inserts were received for i, event in enumerate(inserts): self.assertNotIn('old_image', event) item_id = 'testId%d' % i matching = [i for i in inserts if i['new_image']['id'] == item_id][0] self.assertEqual({'id': item_id, 'data': 'foobar123'}, matching['new_image']) # assert that all updates were received def assert_updates(expected_updates, modifies): def found(update): for modif in modifies: if modif['old_image']['id'] == update['id']: self.assertEqual(modif['old_image'], {'id': update['id'], 'data': update['old']}) self.assertEqual(modif['new_image'], {'id': update['id'], 'data': update['new']}) return True for update in expected_updates: self.assertTrue(found(update)) updates1 = [ {'id': 'testId6', 'old': 'foobar123', 'new': 'foobar123_updated1'}, {'id': 'testId7', 'old': 'foobar123', 'new': 'foobar123_updated1'} ] updates2 = [ {'id': 'testId6', 'old': 'foobar123_updated1', 'new': 'foobar123_updated2'}, {'id': 'testId7', 'old': 'foobar123_updated1', 'new': 'foobar123_updated2'}, {'id': 'testId8', 'old': 'foobar123', 'new': 'foobar123_updated2'} ] assert_updates(updates1, modifies[:2]) assert_updates(updates2, modifies[2:]) # assert that all removes were received for i, event in enumerate(removes): self.assertNotIn('new_image', event) item_id = 'testId%d' % i matching = [i for i in removes if i['old_image']['id'] == item_id][0] self.assertEqual({'id': item_id, 'data': 'foobar123'}, matching['old_image']) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=4) # clean up testutil.delete_lambda_function(lambda_ddb_name)
def test_kinesis_lambda_parallelism(self, lambda_client, kinesis_client): function_name = f"lambda_func-{short_uid()}" stream_name = f"test-foobar-{short_uid()}" testutil.create_lambda_function( handler_file=TEST_LAMBDA_PARALLEL_FILE, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36, ) arn = aws_stack.kinesis_stream_arn(stream_name, account_id="000000000000") lambda_client.create_event_source_mapping(EventSourceArn=arn, FunctionName=function_name) def process_records(record): assert record aws_stack.create_kinesis_stream(stream_name, delete=True) kinesis_connector.listen_to_kinesis( stream_name=stream_name, listener_func=process_records, wait_until_started=True, ) kinesis = aws_stack.create_external_boto_client("kinesis") stream_summary = kinesis.describe_stream_summary(StreamName=stream_name) assert 1 == stream_summary["StreamDescriptionSummary"]["OpenShardCount"] num_events_kinesis = 10 # assure async call start = time.perf_counter() kinesis.put_records( Records=[ {"Data": '{"batch": 0}', "PartitionKey": f"test_{i}"} for i in range(0, num_events_kinesis) ], StreamName=stream_name, ) assert (time.perf_counter() - start) < 1 # this should not take more than a second kinesis.put_records( Records=[ {"Data": '{"batch": 1}', "PartitionKey": f"test_{i}"} for i in range(0, num_events_kinesis) ], StreamName=stream_name, ) def get_events(): events = get_lambda_log_events(function_name, regex_filter=r"event.*Records") assert len(events) == 2 return events events = retry(get_events, retries=5) def assertEvent(event, batch_no): assert 10 == len(event["event"]["Records"]) assert "eventID" in event["event"]["Records"][0] assert "eventSourceARN" in event["event"]["Records"][0] assert "eventSource" in event["event"]["Records"][0] assert "eventVersion" in event["event"]["Records"][0] assert "eventName" in event["event"]["Records"][0] assert "invokeIdentityArn" in event["event"]["Records"][0] assert "awsRegion" in event["event"]["Records"][0] assert "kinesis" in event["event"]["Records"][0] assert {"batch": batch_no} == json.loads( base64.b64decode(event["event"]["Records"][0]["kinesis"]["data"]).decode( config.DEFAULT_ENCODING ) ) assertEvent(events[0], 0) assertEvent(events[1], 1) assert (events[1]["executionStart"] - events[0]["executionStart"]) > 5 # cleanup lambda_client.delete_function(FunctionName=function_name) kinesis_client.delete_stream(StreamName=stream_name)
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [ {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}} ]}) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item(TableName=TEST_TABLE_NAME, Key={PARTITION_KEY: {'S': 'testId%s' % i}}, AttributeUpdates={'data': { 'Action': 'PUT', 'Value': {'S': 'foobar123_updated'} }}) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns def check_events(): if len(EVENTS) != num_events: LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' + '(actual/expected): %s/%s') % (len(EVENTS), num_events)) assert len(EVENTS) == num_events event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS] inserts = [e for e in event_items if e.get('__action_type') == 'INSERT'] modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY'] assert len(inserts) == num_put_new_items + num_batch_items assert len(modifies) == num_put_existing_items + num_updates_ddb # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=7, sleep=3) # make sure the we have the right amount of INSERT/MODIFY event types # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == num_events_ddb
def start_test(env=ENV_DEV): try: # setup environment if env == ENV_DEV: infra.start_infra(async=True) time.sleep(6) dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating stream...') aws_stack.create_kinesis_stream(TEST_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True) print("Kinesis consumer initialized.") # create table with stream forwarding config create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list streams and make sure the table stream is there streams = dynamodbstreams.list_streams() event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: event_source_arn = stream['StreamArn'] assert event_source_arn # deploy test lambda script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py')) zip_file = create_lambda_archive(script, get_content=True) create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=event_source_arn) # put items to table num_events = 10 print('Putting %s items to table...' % num_events) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events): table.put_item(Item={ PARTITION_KEY: 'testId123', 'data': 'foobar123' }) print("Waiting some time before finishing test.") time.sleep(10) print('DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' % (len(EVENTS), num_events)) if len(EVENTS) != num_events: print('ERROR receiving DynamoDB updates. Running processes:') print(run("ps aux | grep 'python\|java\|node'")) assert len(EVENTS) == num_events print("Test finished successfully") cleanup(env=env) except KeyboardInterrupt, e: infra.KILLED = True
def start_test(env=ENV_DEV): try: # setup environment if env == ENV_DEV: infra.start_infra(async=True) time.sleep(6) dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating stream...') aws_stack.create_kinesis_stream(TEST_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True) print("Kinesis consumer initialized.") # create table with stream forwarding config create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list streams and make sure the table stream is there streams = dynamodbstreams.list_streams() event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: event_source_arn = stream['StreamArn'] assert event_source_arn # deploy test lambda script = load_file( os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py')) zip_file = create_lambda_archive(script, get_content=True) create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=event_source_arn) # put items to table num_events = 10 print('Putting %s items to table...' % num_events) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events): table.put_item(Item={ PARTITION_KEY: 'testId123', 'data': 'foobar123' }) print("Waiting some time before finishing test.") time.sleep(10) print( 'DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' % (len(EVENTS), num_events)) if len(EVENTS) != num_events: print('ERROR receiving DynamoDB updates. Running processes:') print(run("ps aux | grep 'python\|java\|node'")) assert len(EVENTS) == num_events print("Test finished successfully") cleanup(env=env) except KeyboardInterrupt, e: infra.KILLED = True