def test_non_ascii_chars(self): dynamodb = aws_stack.connect_to_resource('dynamodb') testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_DDB_TABLE_NAME) # write some items containing non-ASCII characters items = { 'id1': { PARTITION_KEY: 'id1', 'data': 'foobar123 ✓' }, 'id2': { PARTITION_KEY: 'id2', 'data': 'foobar123 £' }, 'id3': { PARTITION_KEY: 'id3', 'data': 'foobar123 ¢' } } for k, item in items.items(): table.put_item(Item=item) for item_id in items.keys(): item = table.get_item(Key={PARTITION_KEY: item_id})['Item'] # need to fix up the JSON and convert str to unicode for Python 2 item1 = json_safe(item) item2 = json_safe(items[item_id]) assert item1 == item2
def test_dynamodb_error_injection(): dynamodb = aws_stack.connect_to_resource('dynamodb') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_TABLE_NAME) # by default, no errors test_no_errors = table.put_item(Item={ PARTITION_KEY: short_uid(), 'data': 'foobar123' }) assert_equal(test_no_errors['ResponseMetadata']['HTTPStatusCode'], 200) # with a probability of 1, always throw errors config.DYNAMODB_ERROR_PROBABILITY = 1.0 assert_raises(ClientError, table.put_item, Item={ PARTITION_KEY: short_uid(), 'data': 'foobar123' }) # reset probability to zero config.DYNAMODB_ERROR_PROBABILITY = 0.0
def test_kinesis_lambda_ddb_streams(env=ENV_DEV): dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating stream...') aws_stack.create_kinesis_stream(TEST_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True) print("Kinesis consumer initialized.") # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list streams and make sure the table stream is there streams = dynamodbstreams.list_streams() event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: event_source_arn = stream['StreamArn'] assert event_source_arn # deploy test lambda script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py')) zip_file = testutil.create_lambda_archive(script, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=event_source_arn) # put items to table num_events = 10 print('Putting %s items to table...' % num_events) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events): table.put_item(Item={ PARTITION_KEY: 'testId123', 'data': 'foobar123' }) print("Waiting some time before finishing test.") time.sleep(5) print('DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' % (len(EVENTS), num_events)) if len(EVENTS) != num_events: print('ERROR receiving DynamoDB updates. Running processes:') print(run("ps aux | grep 'python\|java\|node'")) assert len(EVENTS) == num_events
def test_time_to_live(self): dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_client = aws_stack.connect_to_service('dynamodb') testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME_3, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_DDB_TABLE_NAME_3) # Insert some items to the table items = { 'id1': {PARTITION_KEY: 'id1', 'data': 'IT IS'}, 'id2': {PARTITION_KEY: 'id2', 'data': 'TIME'}, 'id3': {PARTITION_KEY: 'id3', 'data': 'TO LIVE!'} } for k, item in items.items(): table.put_item(Item=item) # Describe TTL when still unset. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED' # Enable TTL for given table response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, True) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is True # Describe TTL status after being enabled. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED' # Disable TTL for given table response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, False) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is False # Describe TTL status after being disabled. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED' # Enable TTL for given table again response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, True) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is True # Describe TTL status after being enabled again. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED' # Clean up table dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_3)
def test_time_to_live(self): dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_client = aws_stack.connect_to_service('dynamodb') testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME_3, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_DDB_TABLE_NAME_3) # Insert some items to the table items = { 'id1': {PARTITION_KEY: 'id1', 'data': 'IT IS'}, 'id2': {PARTITION_KEY: 'id2', 'data': 'TIME'}, 'id3': {PARTITION_KEY: 'id3', 'data': 'TO LIVE!'} } for k, item in items.items(): table.put_item(Item=item) # Describe TTL when still unset. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED' # Enable TTL for given table response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, True) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is True # Describe TTL status after being enabled. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED' # Disable TTL for given table response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, False) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is False # Describe TTL status after being disabled. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED' # Enable TTL for given table again response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, True) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is True # Describe TTL status after being enabled again. response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3) assert response.status_code == 200 assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED' # Clean up table dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_3)
def test_region_replacement(self): dynamodb = aws_stack.connect_to_resource('dynamodb') testutil.create_dynamodb_table( TEST_DDB_TABLE_NAME_4, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES' ) table = dynamodb.Table(TEST_DDB_TABLE_NAME_4) expected_arn_prefix = 'arn:aws:dynamodb:' + aws_stack.get_local_region() assert table.table_arn.startswith(expected_arn_prefix) assert table.latest_stream_arn.startswith(expected_arn_prefix)
def test_large_data_download(self): dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_client = aws_stack.connect_to_service('dynamodb') testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME_2, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_DDB_TABLE_NAME_2) # Create a large amount of items num_items = 20 for i in range(0, num_items): item = {PARTITION_KEY: 'id%s' % i, 'data1': 'foobar123 ' * 1000} table.put_item(Item=item) # Retrieve the items. The data will be transmitted to the client with chunked transfer encoding result = table.scan(TableName=TEST_DDB_TABLE_NAME_2) assert len(result['Items']) == num_items # Clean up dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_2)
def test_large_data_download(self): dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_client = aws_stack.connect_to_service('dynamodb') testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME_2, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_DDB_TABLE_NAME_2) # Create a large amount of items num_items = 20 for i in range(0, num_items): item = {PARTITION_KEY: 'id%s' % i, 'data1': 'foobar123 ' * 1000} table.put_item(Item=item) # Retrieve the items. The data will be transmitted to the client with chunked transfer encoding result = table.scan(TableName=TEST_DDB_TABLE_NAME_2) assert len(result['Items']) == num_items # Clean up dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_2)
def test_dynamodb_error_injection(): if not do_run(): return dynamodb = aws_stack.connect_to_resource('dynamodb') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_TABLE_NAME) # by default, no errors test_no_errors = table.put_item(Item={PARTITION_KEY: short_uid(), 'data': 'foobar123'}) assert_equal(test_no_errors['ResponseMetadata']['HTTPStatusCode'], 200) # with a probability of 1, always throw errors config.DYNAMODB_ERROR_PROBABILITY = 1.0 assert_raises(ClientError, table.put_item, Item={PARTITION_KEY: short_uid(), 'data': 'foobar123'}) # reset probability to zero config.DYNAMODB_ERROR_PROBABILITY = 0.0
def test_non_ascii_chars(self): dynamodb = aws_stack.connect_to_resource('dynamodb') testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME, partition_key=PARTITION_KEY) table = dynamodb.Table(TEST_DDB_TABLE_NAME) # write some items containing non-ASCII characters items = { 'id1': {PARTITION_KEY: 'id1', 'data': 'foobar123 ✓'}, 'id2': {PARTITION_KEY: 'id2', 'data': 'foobar123 £'}, 'id3': {PARTITION_KEY: 'id3', 'data': 'foobar123 ¢'} } for k, item in items.items(): table.put_item(Item=item) for item_id in items.keys(): item = table.get_item(Key={PARTITION_KEY: item_id})['Item'] # need to fix up the JSON and convert str to unicode for Python 2 item1 = json_safe(item) item2 = json_safe(items[item_id]) assert item1 == item2
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info("Kinesis consumer initialized.") # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # put items to table num_events_ddb = 10 LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events_ddb - 3): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [ {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123'}}} ]}) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info("Waiting some time before finishing test.") time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns if len(EVENTS) != num_events: LOGGER.warning('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events)) assert len(EVENTS) == num_events # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == 10
def test_kinesis_lambda_ddb_streams(): env = ENV_DEV ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodb_service = aws_stack.connect_to_service('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) print("Kinesis consumer initialized.") # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda (Python) connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(TEST_LAMBDA_PYTHON, get_content=True, libs=['localstack'], runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda (Python) connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM_PY, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda (Java) connected to Kinesis Stream zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_STREAM_JAVA, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_JAVA8, handler='com.atlassian.localstack.sample.KinesisHandler') if use_docker(): # deploy test lambda (Node.js) connected to Kinesis Stream zip_file = testutil.create_lambda_archive( TEST_LAMBDA_NODEJS, get_content=True, runtime=LAMBDA_RUNTIME_NODEJS) kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_STREAM_JS, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_NODEJS) # put items to table num_events_ddb = 10 print('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events_ddb): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # put items to stream num_events_kinesis = 10 print('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 print("Waiting some time before finishing test.") time.sleep(2) num_events = num_events_ddb + num_events_kinesis print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events)) assert len(EVENTS) == num_events
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item( RequestItems={ TEST_TABLE_NAME: [{ 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 £' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢' } } }] }) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item( TableName=TEST_TABLE_NAME, Key={PARTITION_KEY: { 'S': 'testId%s' % i }}, AttributeUpdates={ 'data': { 'Action': 'PUT', 'Value': { 'S': 'foobar123_updated' } } }) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe( TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns def check_events(): if len(EVENTS) != num_events: LOGGER.warning( ('DynamoDB and Kinesis updates retrieved ' + '(actual/expected): %s/%s') % (len(EVENTS), num_events)) assert len(EVENTS) == num_events event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS] inserts = [ e for e in event_items if e.get('__action_type') == 'INSERT' ] modifies = [ e for e in event_items if e.get('__action_type') == 'MODIFY' ] assert len(inserts) == num_put_new_items + num_batch_items assert len(modifies) == num_put_existing_items + num_updates_ddb # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=7, sleep=3) # make sure the we have the right amount of INSERT/MODIFY event types # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == num_events_ddb
def test_kinesis_lambda_ddb_streams(): env = ENV_DEV dynamodb = aws_stack.connect_to_resource('dynamodb', env=env) dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env) kinesis = aws_stack.connect_to_service('kinesis', env=env) print('Creating test streams...') aws_stack.create_kinesis_stream(TEST_STREAM_NAME) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True) print("Kinesis consumer initialized.") # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, env=env, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py')) zip_file = testutil.create_lambda_archive(script, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=ddb_event_source_arn) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_SOURCE_STREAM_NAME, zip_file=zip_file, event_source_arn=kinesis_event_source_arn) # put items to table num_events_ddb = 10 print('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events_ddb): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # put items to stream num_events_kinesis = 10 print('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) print("Waiting some time before finishing test.") time.sleep(4) num_events = num_events_ddb + num_events_kinesis print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' % (len(EVENTS), num_events)) if len(EVENTS) != num_events: print('ERROR receiving DynamoDB updates.') assert len(EVENTS) == num_events
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [ {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}} ]}) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item(TableName=TEST_TABLE_NAME, Key={PARTITION_KEY: {'S': 'testId%s' % i}}, AttributeUpdates={'data': { 'Action': 'PUT', 'Value': {'S': 'foobar123_updated'} }}) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns def check_events(): if len(EVENTS) != num_events: LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' + '(actual/expected): %s/%s') % (len(EVENTS), num_events)) assert len(EVENTS) == num_events event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS] inserts = [e for e in event_items if e.get('__action_type') == 'INSERT'] modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY'] assert len(inserts) == num_put_new_items + num_batch_items assert len(modifies) == num_put_existing_items + num_updates_ddb # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=7, sleep=3) # make sure the we have the right amount of INSERT/MODIFY event types # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == num_events_ddb