def checkpoint(self, checkpointer, sequence_number=None, sub_sequence_number=None):
def do_checkpoint():
checkpointer.checkpoint(sequence_number, sub_sequence_number)
try:
retry(do_checkpoint, retries=CHECKPOINT_RETRIES, sleep=CHECKPOINT_SLEEP_SECS)
except Exception as e:
LOGGER.warning('Unable to checkpoint Kinesis after retries: %s' % e)
def test_list_stack_resources_returns_queue_urls(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_2))
cloudformation.create_stack(StackName=TEST_STACK_NAME_2, TemplateBody=template)
def check_stack():
stack = get_stack_details(TEST_STACK_NAME_2)
assert stack['StackStatus'] == 'CREATE_COMPLETE'
retry(check_stack, retries=3, sleep=2)
list_stack_summaries = list_stack_resources(TEST_STACK_NAME_2)
queue_urls = get_queue_urls()
for resource in list_stack_summaries:
assert resource['PhysicalResourceId'] in queue_urls
def test_apply_template(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1))
# deploy template
cloudformation.create_stack(StackName=TEST_STACK_NAME, TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(TEST_STACK_NAME)
assert stack['StackStatus'] == 'CREATE_COMPLETE'
retry(check_stack, retries=3, sleep=2)
# assert that bucket has been created
assert bucket_exists('cf-test-bucket-1')
# assert that queue has been created
assert queue_exists('cf-test-queue-1')
# assert that stream has been created
assert stream_exists('cf-test-stream-1')
# assert that queue has been created
resource = describe_stack_resource(TEST_STACK_NAME, 'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = '-kclapp2'
table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_service('dynamodb', client=True)
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=stream_name + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(stream_name, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON, libs=TEST_LAMBDA_LIBS, func_name=TEST_LAMBDA_NAME_DDB,
event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True)
# submit a batch with writes
dynamodb.batch_write_item(RequestItems={table_name: [
{'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId0'}, 'data': {'S': 'foobar123'}}}},
{'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId1'}, 'data': {'S': 'foobar123'}}}},
{'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId2'}, 'data': {'S': 'foobar123'}}}}
]})
# submit a batch with writes and deletes
dynamodb.batch_write_item(RequestItems={table_name: [
{'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId3'}, 'data': {'S': 'foobar123'}}}},
{'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId4'}, 'data': {'S': 'foobar123'}}}},
{'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId5'}, 'data': {'S': 'foobar123'}}}},
{'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId0'}}}},
{'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId1'}}}},
{'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId2'}}}},
]})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{'Put': {'TableName': table_name,
'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123'}}}},
{'Put': {'TableName': table_name,
'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123'}}}},
{'Put': {'TableName': table_name,
'Item': {PARTITION_KEY: {'S': 'testId8'}, 'data': {'S': 'foobar123'}}}},
{'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId3'}}}},
{'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId4'}}}},
{'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId5'}}}},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{'Put': {'TableName': table_name,
'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123_updated1'}}}},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{'Put': {'TableName': table_name,
'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123_updated1'}}}},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId6'}},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {'#0': 'data'},
'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}},
{'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId7'}},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {'#0': 'data'},
'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}},
{'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId8'}},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {'#0': 'data'},
'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}},
])
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
LOGGER.warning(('DynamoDB updates retrieved (actual/expected): %s/%s') %
(len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [json.loads(base64.b64decode(e['data'])) for e in events]
# make sure the we have the right amount of expected event types
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
removes = [e for e in event_items if e.get('__action_type') == 'REMOVE']
self.assertEqual(len(inserts), num_insert)
self.assertEqual(len(modifies), num_modify)
self.assertEqual(len(removes), num_delete)
# assert that all inserts were received
for i, event in enumerate(inserts):
self.assertNotIn('old_image', event)
item_id = 'testId%d' % i
matching = [i for i in inserts if i['new_image']['id'] == item_id][0]
self.assertEqual(matching['new_image'], {'id': item_id, 'data': 'foobar123'})
# assert that all updates were received
def assert_updates(expected_updates, modifies):
def found(update):
for modif in modifies:
if modif['old_image']['id'] == update['id']:
self.assertEqual(modif['old_image'], {'id': update['id'], 'data': update['old']})
self.assertEqual(modif['new_image'], {'id': update['id'], 'data': update['new']})
return True
for update in expected_updates:
self.assertTrue(found(update))
updates1 = [
{'id': 'testId6', 'old': 'foobar123', 'new': 'foobar123_updated1'},
{'id': 'testId7', 'old': 'foobar123', 'new': 'foobar123_updated1'}
]
updates2 = [
{'id': 'testId6', 'old': 'foobar123_updated1', 'new': 'foobar123_updated2'},
{'id': 'testId7', 'old': 'foobar123_updated1', 'new': 'foobar123_updated2'},
{'id': 'testId8', 'old': 'foobar123', 'new': 'foobar123_updated2'}
]
assert_updates(updates1, modifies[:2])
assert_updates(updates2, modifies[2:])
# assert that all removes were received
for i, event in enumerate(removes):
self.assertNotIn('new_image', event)
item_id = 'testId%d' % i
matching = [i for i in removes if i['old_image']['id'] == item_id][0]
self.assertEqual(matching['old_image'], {'id': item_id, 'data': 'foobar123'})
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=4)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_put_events_into_event_bus(self):
queue_name = 'queue-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
bus_name_1 = 'bus1-{}'.format(short_uid())
bus_name_2 = 'bus2-{}'.format(short_uid())
sqs_client = aws_stack.connect_to_service('sqs')
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
self.events_client.create_event_bus(Name=bus_name_1)
resp = self.events_client.create_event_bus(Name=bus_name_2)
print(resp)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name_1,
)
self.events_client.put_targets(Rule=rule_name,
EventBusName=bus_name_1,
Targets=[{
'Id': target_id,
'Arn': resp.get('EventBusArn')
}])
self.events_client.put_targets(Rule=rule_name,
EventBusName=bus_name_2,
Targets=[{
'Id': target_id,
'Arn': queue_arn
}])
self.events_client.put_events(
Entries=[{
'EventBusName': bus_name_1,
'Source': TEST_EVENT_PATTERN['Source'][0],
'DetailType': TEST_EVENT_PATTERN['detail-type'][0],
'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0])
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp['Messages']
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(len(messages), 1)
actual_event = json.loads(messages[0]['Body'])
self.assertIsValidEvent(actual_event)
self.assertEqual(actual_event['detail'],
TEST_EVENT_PATTERN['Detail'][0])
# clean up
sqs_client.delete_queue(QueueUrl=queue_url)
self.events_client.remove_targets(Rule=rule_name,
EventBusName=bus_name_1,
Ids=[target_id],
Force=True)
self.events_client.remove_targets(Rule=rule_name,
EventBusName=bus_name_2,
Ids=[target_id],
Force=True)
self.events_client.delete_rule(Name=rule_name,
EventBusName=bus_name_1,
Force=True)
self.events_client.delete_event_bus(Name=bus_name_1)
self.events_client.delete_event_bus(Name=bus_name_2)
def test_scheduled_expression_events(self):
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
return 200
local_port = get_free_tcp_port()
proxy = start_proxy(local_port,
backend_url=None,
update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
topic_name = 'topic-{}'.format(short_uid())
queue_name = 'queue-{}'.format(short_uid())
fifo_queue_name = 'queue-{}.fifo'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
endpoint = '{}://{}:{}'.format(get_service_protocol(),
config.LOCALSTACK_HOSTNAME, local_port)
sm_role_arn = aws_stack.role_arn('sfn_role')
sm_name = 'state-machine-{}'.format(short_uid())
topic_target_id = 'target-{}'.format(short_uid())
sm_target_id = 'target-{}'.format(short_uid())
queue_target_id = 'target-{}'.format(short_uid())
fifo_queue_target_id = 'target-{}'.format(short_uid())
events = []
state_machine_definition = """
{
"StartAt": "Hello",
"States": {
"Hello": {
"Type": "Pass",
"Result": "World",
"End": true
}
}
}
"""
state_machine_arn = self.sfn_client.create_state_machine(
name=sm_name,
definition=state_machine_definition,
roleArn=sm_role_arn)['stateMachineArn']
topic_arn = self.sns_client.create_topic(Name=topic_name)['TopicArn']
self.sns_client.subscribe(TopicArn=topic_arn,
Protocol='http',
Endpoint=endpoint)
queue_url = self.sqs_client.create_queue(
QueueName=queue_name)['QueueUrl']
fifo_queue_url = self.sqs_client.create_queue(
QueueName=fifo_queue_name, Attributes={'FifoQueue':
'true'})['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
fifo_queue_arn = aws_stack.sqs_queue_arn(fifo_queue_name)
event = {'env': 'testing'}
self.events_client.put_rule(Name=rule_name,
ScheduleExpression='rate(1 minutes)')
self.events_client.put_targets(Rule=rule_name,
Targets=[{
'Id': topic_target_id,
'Arn': topic_arn,
'Input': json.dumps(event)
}, {
'Id': sm_target_id,
'Arn': state_machine_arn,
'Input': json.dumps(event)
}, {
'Id': queue_target_id,
'Arn': queue_arn,
'Input': json.dumps(event)
}, {
'Id': fifo_queue_target_id,
'Arn': fifo_queue_arn,
'Input': json.dumps(event),
'SqsParameters': {
'MessageGroupId': '123'
}
}])
def received(q_urls):
# state machine got executed
executions = self.sfn_client.list_executions(
stateMachineArn=state_machine_arn)['executions']
self.assertGreaterEqual(len(executions), 1)
# http endpoint got events
self.assertGreaterEqual(len(events), 2)
notifications = [
event['Message'] for event in events
if event['Type'] == 'Notification'
]
self.assertGreaterEqual(len(notifications), 1)
# get state machine execution detail
execution_arn = executions[0]['executionArn']
execution_input = self.sfn_client.describe_execution(
executionArn=execution_arn)['input']
all_msgs = []
# get message from queue
for url in q_urls:
msgs = self.sqs_client.receive_message(QueueUrl=url).get(
'Messages', [])
self.assertGreaterEqual(len(msgs), 1)
all_msgs.append(msgs[0])
return execution_input, notifications[0], all_msgs
execution_input, notification, msgs_received = retry(
received, retries=5, sleep=15, q_urls=[queue_url, fifo_queue_url])
self.assertEqual(json.loads(notification), event)
self.assertEqual(json.loads(execution_input), event)
for msg_received in msgs_received:
self.assertEqual(json.loads(msg_received['Body']), event)
# clean up
proxy.stop()
self.cleanup(None,
rule_name,
target_ids=[topic_target_id, sm_target_id],
queue_url=queue_url)
self.sns_client.delete_topic(TopicArn=topic_arn)
self.sfn_client.delete_state_machine(stateMachineArn=state_machine_arn)
def test_put_event_with_content_base_rule_in_pattern(self):
queue_name = 'queue-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
sqs_client = aws_stack.connect_to_service('sqs')
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
pattern = {
'Source': [{
'exists': True
}],
'detail-type': [{
'prefix': 'core.app'
}],
'Detail':
json.dumps({
'decription': ['this-is-event-details'],
'amount': [200],
'salary': [2000, 4000],
'env': ['dev', 'prod'],
'user': ['user1', 'user2', 'user3'],
'admins': ['skyli', {
'prefix': 'hey'
}, {
'prefix': 'ad'
}],
'test1': [{
'anything-but': 200
}],
'test2': [{
'anything-but': 'test2'
}],
'test3': [{
'anything-but': ['test3', 'test33']
}],
'test4': [{
'anything-but': {
'prefix': 'test4'
}
}],
'ip': [{
'cidr': '10.102.1.0/24'
}],
'num-test1': [{
'numeric': ['<', 200]
}],
'num-test2': [{
'numeric': ['<=', 200]
}],
'num-test3': [{
'numeric': ['>', 200]
}],
'num-test4': [{
'numeric': ['>=', 200]
}],
'num-test5': [{
'numeric': ['>=', 200, '<=', 500]
}],
'num-test6': [{
'numeric': ['>', 200, '<', 500]
}],
'num-test7': [{
'numeric': ['>=', 200, '<', 500]
}]
})
}
event = {
'EventBusName':
TEST_EVENT_BUS_NAME,
'Source':
'core.update-account-command',
'DetailType':
'core.app.backend',
'Detail':
json.dumps({
'decription': 'this-is-event-details',
'amount': 200,
'salary': 2000,
'env': 'prod',
'user': ['user4', 'user3'],
'admins': 'admin',
'test1': 300,
'test2': 'test22',
'test3': 'test333',
'test4': 'this test4',
'ip': '10.102.1.100',
'num-test1': 100,
'num-test2': 200,
'num-test3': 300,
'num-test4': 200,
'num-test5': 500,
'num-test6': 300,
'num-test7': 300
})
}
self.events_client.create_event_bus(Name=TEST_EVENT_BUS_NAME)
self.events_client.put_rule(Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
EventPattern=json.dumps(pattern))
self.events_client.put_targets(Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Targets=[{
'Id': target_id,
'Arn': queue_arn,
'InputPath': '$.detail'
}])
self.events_client.put_events(Entries=[event])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp.get('Messages')
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(len(messages), 1)
self.assertEqual(json.loads(messages[0].get('Body')),
json.loads(event['Detail']))
event_details = json.loads(event['Detail'])
event_details['admins'] = 'not_admin'
event['Detail'] = json.dumps(event_details)
self.events_client.put_events(Entries=[event])
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(messages, None)
# clean up
self.cleanup(TEST_EVENT_BUS_NAME,
rule_name,
target_id,
queue_url=queue_url)
def test_schedule_expression_event_stepfunction(self):
state_machine_name = 'state-machine-{}'.format(short_uid())
role_name = 'role-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
state_machine_definition = """
{
"StartAt": "Hello",
"States": {
"Hello": {
"Type": "Pass",
"Result": "World",
"End": true
}
}
}
"""
assume_role_policy_document = """
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "events.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
"""
state_machine_role_arn = self.iam_client.create_role(
RoleName=role_name,
AssumeRolePolicyDocument=assume_role_policy_document
)['Role']['Arn']
state_machine_arn = self.stepfunctions_client.create_state_machine(
name=state_machine_name,
definition=state_machine_definition,
roleArn=state_machine_role_arn)['stateMachineArn']
event = {'env': 'testing'}
self.events_client.put_rule(Name=rule_name,
ScheduleExpression='rate(1 minutes)')
self.events_client.put_targets(Rule=rule_name,
Targets=[{
'Id': target_id,
'Arn': state_machine_arn,
'Input': json.dumps(event)
}])
def check_executions():
executions = self.stepfunctions_client.list_executions(
stateMachineArn=state_machine_arn)['executions']
self.assertGreaterEqual(len(executions), 1)
execution_arn = executions[0]['executionArn']
execution_input = self.stepfunctions_client.describe_execution(
executionArn=execution_arn)['input']
self.assertEqual(execution_input, json.dumps(event))
retry(check_executions, retries=2, sleep=40)
self.events_client.remove_targets(Rule=rule_name,
Ids=[target_id],
Force=True)
self.events_client.delete_rule(Name=rule_name, Force=True)
self.iam_client.delete_role(RoleName=role_name)
self.stepfunctions_client.delete_state_machine(
stateMachineArn=state_machine_arn)
def test_create_delete_stack(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
cf_client = aws_stack.connect_to_service('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
apigateway = aws_stack.connect_to_service('apigateway')
template = template_deployer.template_to_json(
load_file(TEST_TEMPLATE_1))
# deploy template
stack_name = 'stack-%s' % short_uid()
cloudformation.create_stack(StackName=stack_name,
TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(stack_name)
self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE')
retry(check_stack, retries=3, sleep=2)
# assert that resources have been created
assert bucket_exists('cf-test-bucket-1')
assert queue_exists('cf-test-queue-1')
topic_arn = topic_exists('%s-test-topic-1-1' % stack_name)
assert topic_arn
assert stream_exists('cf-test-stream-1')
resource = describe_stack_resource(stack_name,
'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
# assert that tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(
tags, [{
'Key': 'foobar',
'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')
}])
tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags']
self.assertEqual(
tags, [{
'Key': 'foo',
'Value': 'cf-test-bucket-1'
}, {
'Key': 'bar',
'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')
}])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [
s for s in subs
if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']
]
self.assertEqual(len(subs), 1)
self.assertIn(
':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name),
subs[0]['TopicArn'])
# assert that subscription attributes are added properly
attrs = sns.get_subscription_attributes(
SubscriptionArn=subs[0]['SubscriptionArn'])['Attributes']
self.assertEqual(
attrs, {
'Endpoint': subs[0]['Endpoint'],
'Protocol': 'sqs',
'SubscriptionArn': subs[0]['SubscriptionArn'],
'TopicArn': subs[0]['TopicArn'],
'FilterPolicy': json.dumps({'eventType': ['created']})
})
# assert that Gateway responses have been created
test_api_name = 'test-api'
api = [
a for a in apigateway.get_rest_apis()['items']
if a['name'] == test_api_name
][0]
responses = apigateway.get_gateway_responses(
restApiId=api['id'])['items']
self.assertEqual(len(responses), 2)
types = [r['responseType'] for r in responses]
self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
# delete the stack
cf_client.delete_stack(StackName=stack_name)
# assert that resources have been deleted
assert not bucket_exists('cf-test-bucket-1')
assert not queue_exists('cf-test-queue-1')
assert not topic_exists('%s-test-topic-1-1' % stack_name)
retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def test_put_event_with_content_base_rule_in_pattern(self):
queue_name = "queue-{}".format(short_uid())
rule_name = "rule-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
sqs_client = aws_stack.create_external_boto_client("sqs")
queue_url = sqs_client.create_queue(QueueName=queue_name)["QueueUrl"]
queue_arn = aws_stack.sqs_queue_arn(queue_name)
pattern = {
"Source": [{
"exists": True
}],
"detail-type": [{
"prefix": "core.app"
}],
"Detail": {
"decription": ["this-is-event-details"],
"amount": [200],
"salary": [2000, 4000],
"env": ["dev", "prod"],
"user": ["user1", "user2", "user3"],
"admins": ["skyli", {
"prefix": "hey"
}, {
"prefix": "ad"
}],
"test1": [{
"anything-but": 200
}],
"test2": [{
"anything-but": "test2"
}],
"test3": [{
"anything-but": ["test3", "test33"]
}],
"test4": [{
"anything-but": {
"prefix": "test4"
}
}],
"ip": [{
"cidr": "10.102.1.0/24"
}],
"num-test1": [{
"numeric": ["<", 200]
}],
"num-test2": [{
"numeric": ["<=", 200]
}],
"num-test3": [{
"numeric": [">", 200]
}],
"num-test4": [{
"numeric": [">=", 200]
}],
"num-test5": [{
"numeric": [">=", 200, "<=", 500]
}],
"num-test6": [{
"numeric": [">", 200, "<", 500]
}],
"num-test7": [{
"numeric": [">=", 200, "<", 500]
}],
},
}
event = {
"EventBusName":
TEST_EVENT_BUS_NAME,
"Source":
"core.update-account-command",
"DetailType":
"core.app.backend",
"Detail":
json.dumps({
"decription": "this-is-event-details",
"amount": 200,
"salary": 2000,
"env": "prod",
"user": "******",
"admins": "admin",
"test1": 300,
"test2": "test22",
"test3": "test333",
"test4": "this test4",
"ip": "10.102.1.100",
"num-test1": 100,
"num-test2": 200,
"num-test3": 300,
"num-test4": 200,
"num-test5": 500,
"num-test6": 300,
"num-test7": 300,
}),
}
self.events_client.create_event_bus(Name=TEST_EVENT_BUS_NAME)
self.events_client.put_rule(
Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
EventPattern=json.dumps(pattern),
)
self.events_client.put_targets(
Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Targets=[{
"Id": target_id,
"Arn": queue_arn,
"InputPath": "$.detail"
}],
)
self.events_client.put_events(Entries=[event])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp.get("Messages")
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(1, len(messages))
self.assertEqual(json.loads(event["Detail"]),
json.loads(messages[0].get("Body")))
event_details = json.loads(event["Detail"])
event_details["admins"] = "no"
event["Detail"] = json.dumps(event_details)
self.events_client.put_events(Entries=[event])
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertIsNone(messages)
# clean up
self.cleanup(TEST_EVENT_BUS_NAME,
rule_name,
target_id,
queue_url=queue_url)
def test_put_events_with_target_sns(self):
queue_name = 'test-%s' % short_uid()
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
bus_name = 'bus-{}'.format(short_uid())
sns_client = aws_stack.connect_to_service('sns')
sqs_client = aws_stack.connect_to_service('sqs')
topic_name = 'topic-{}'.format(short_uid())
topic_arn = sns_client.create_topic(Name=topic_name)['TopicArn']
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
sns_client.subscribe(TopicArn=topic_arn, Protocol='sqs', Endpoint=queue_arn)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN)
)
rs = self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[
{
'Id': target_id,
'Arn': topic_arn
}
]
)
self.assertIn('FailedEntryCount', rs)
self.assertIn('FailedEntries', rs)
self.assertEqual(rs['FailedEntryCount'], 0)
self.assertEqual(rs['FailedEntries'], [])
self.events_client.put_events(
Entries=[{
'EventBusName': bus_name,
'Source': TEST_EVENT_PATTERN['Source'][0],
'DetailType': TEST_EVENT_PATTERN['detail-type'][0],
'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0])
}]
)
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp['Messages']
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(len(messages), 1)
actual_event = json.loads(messages[0]['Body']).get('Message')
self.assertIsValidEvent(actual_event)
self.assertEqual(json.loads(actual_event).get('detail'), TEST_EVENT_PATTERN['Detail'][0])
# clean up
sns_client.delete_topic(TopicArn=topic_arn)
self.cleanup(bus_name, rule_name, target_id, queue_url=queue_url)
def _await_stack_status(stack_name, expected_status, retries=3, sleep=2):
def check_stack():
stack = get_stack_details(stack_name)
assert stack['StackStatus'] == expected_status
return stack
return retry(check_stack, retries, sleep)
def test_create_delete_stack(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
cf_client = aws_stack.connect_to_service('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
apigateway = aws_stack.connect_to_service('apigateway')
template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1))
# deploy template
stack_name = 'stack-%s' % short_uid()
cloudformation.create_stack(StackName=stack_name, TemplateBody=template)
_await_stack_completion(stack_name)
# assert that resources have been created
assert bucket_exists('cf-test-bucket-1')
queue_url = queue_exists('cf-test-queue-1')
assert queue_url
topic_arn = topic_exists('%s-test-topic-1-1' % stack_name)
assert topic_arn
assert stream_exists('cf-test-stream-1')
resource = describe_stack_resource(stack_name, 'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
assert ssm_param_exists('cf-test-param-1')
# assert that tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(tags, [{'Key': 'foobar', 'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')}])
tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags']
self.assertEqual(tags, [
{'Key': 'foo', 'Value': 'cf-test-bucket-1'},
{'Key': 'bar', 'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')}
])
queue_tags = sqs.list_queue_tags(QueueUrl=queue_url)
self.assertIn('Tags', queue_tags)
self.assertEqual(queue_tags['Tags'], {'key1': 'value1', 'key2': 'value2'})
# assert that bucket notifications have been created
notifications = s3.get_bucket_notification_configuration(Bucket='cf-test-bucket-1')
self.assertIn('QueueConfigurations', notifications)
self.assertIn('LambdaFunctionConfigurations', notifications)
self.assertEqual(notifications['QueueConfigurations'][0]['QueueArn'], 'aws:arn:sqs:test:testqueue')
self.assertEqual(notifications['QueueConfigurations'][0]['Events'], ['s3:ObjectDeleted:*'])
self.assertEqual(
notifications['LambdaFunctionConfigurations'][0]['LambdaFunctionArn'],
'aws:arn:lambda:test:testfunc'
)
self.assertEqual(notifications['LambdaFunctionConfigurations'][0]['Events'], ['s3:ObjectCreated:*'])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [s for s in subs if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']]
self.assertEqual(len(subs), 1)
self.assertIn(':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name), subs[0]['TopicArn'])
# assert that subscription attributes are added properly
attrs = sns.get_subscription_attributes(SubscriptionArn=subs[0]['SubscriptionArn'])['Attributes']
self.assertEqual(attrs, {'Endpoint': subs[0]['Endpoint'], 'Protocol': 'sqs',
'SubscriptionArn': subs[0]['SubscriptionArn'], 'TopicArn': subs[0]['TopicArn'],
'FilterPolicy': json.dumps({'eventType': ['created']})})
# assert that Gateway responses have been created
test_api_name = 'test-api'
api = [a for a in apigateway.get_rest_apis()['items'] if a['name'] == test_api_name][0]
responses = apigateway.get_gateway_responses(restApiId=api['id'])['items']
self.assertEqual(len(responses), 2)
types = [r['responseType'] for r in responses]
self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
# delete the stack
cf_client.delete_stack(StackName=stack_name)
# assert that resources have been deleted
assert not bucket_exists('cf-test-bucket-1')
assert not queue_exists('cf-test-queue-1')
assert not topic_exists('%s-test-topic-1-1' % stack_name)
retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def test_put_events_with_target_sqs(self):
queue_name = 'queue-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
sqs_client = aws_stack.connect_to_service('sqs')
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
self.events_client.create_event_bus(
Name=TEST_EVENT_BUS_NAME
)
self.events_client.put_rule(
Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
EventPattern=json.dumps(TEST_EVENT_PATTERN)
)
rs = self.events_client.put_targets(
Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Targets=[
{
'Id': target_id,
'Arn': queue_arn
}
]
)
self.assertIn('FailedEntryCount', rs)
self.assertIn('FailedEntries', rs)
self.assertEqual(rs['FailedEntryCount'], 0)
self.assertEqual(rs['FailedEntries'], [])
self.events_client.put_events(
Entries=[{
'EventBusName': TEST_EVENT_BUS_NAME,
'Source': TEST_EVENT_PATTERN['Source'],
'DetailType': TEST_EVENT_PATTERN['DetailType'],
'Detail': TEST_EVENT_PATTERN['Detail']
}]
)
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp['Messages']
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]['Body'], TEST_EVENT_PATTERN['Detail'])
# clean up
sqs_client.delete_queue(QueueUrl=queue_url)
self.events_client.remove_targets(
Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Ids=[target_id],
Force=True
)
self.events_client.delete_rule(
Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Force=True
)
self.events_client.delete_event_bus(
Name=TEST_EVENT_BUS_NAME
)
def test_sqs_batch_lambda_forward(self):
sqs = aws_stack.connect_to_service('sqs')
lambda_api = aws_stack.connect_to_service('lambda')
lambda_name_queue_batch = 'lambda_queue_batch-%s' % short_uid()
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(lambda_name_queue_batch)
queue_url = sqs_queue_info['QueueUrl']
resp = testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON_ECHO,
func_name=lambda_name_queue_batch,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27,
libs=TEST_LAMBDA_LIBS)
event_source_id = resp['CreateEventSourceMappingResponse']['UUID']
lambda_api.update_event_source_mapping(UUID=event_source_id,
BatchSize=5)
messages_to_send = [{
'Id': 'message{:02d}'.format(i),
'MessageBody': 'msgBody{:02d}'.format(i),
'MessageAttributes': {
'CustomAttribute': {
'DataType': 'String',
'StringValue': 'CustomAttributeValue{:02d}'.format(i)
}
}
} for i in range(1, 12)]
# send 11 messages (which should get split into 3 batches)
sqs.send_message_batch(QueueUrl=queue_url,
Entries=messages_to_send[:10])
sqs.send_message(
QueueUrl=queue_url,
MessageBody=messages_to_send[10]['MessageBody'],
MessageAttributes=messages_to_send[10]['MessageAttributes'])
def wait_for_done():
attributes = sqs.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=[
'ApproximateNumberOfMessages',
'ApproximateNumberOfMessagesDelayed',
'ApproximateNumberOfMessagesNotVisible'
],
)['Attributes']
msg_count = int(attributes.get('ApproximateNumberOfMessages'))
self.assertEqual(msg_count, 0, 'expecting queue to be empty')
delayed_count = int(
attributes.get('ApproximateNumberOfMessagesDelayed'))
if delayed_count != 0:
LOGGER.warning(
'SQS delayed message count (actual/expected): %s/%s' %
(delayed_count, 0))
not_visible_count = int(
attributes.get('ApproximateNumberOfMessagesNotVisible'))
if not_visible_count != 0:
LOGGER.warning(
'SQS messages not visible (actual/expected): %s/%s' %
(not_visible_count, 0))
self.assertEqual(delayed_count, 0, 'no messages waiting for retry')
self.assertEqual(delayed_count + not_visible_count, 0,
'no in flight messages')
# wait for the queue to drain (max 60s)
retry(wait_for_done, retries=12, sleep=5.0)
events = get_lambda_log_events(lambda_name_queue_batch, 10)
self.assertEqual(len(events), 3, 'expected 3 lambda invocations')
testutil.delete_lambda_function(lambda_name_queue_batch)
sqs.delete_queue(QueueUrl=queue_url)
def test_sqs_batch_lambda_forward(lambda_client, sqs_client,
create_lambda_function):
lambda_name_queue_batch = "lambda_queue_batch-%s" % short_uid()
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(lambda_name_queue_batch)
queue_url = sqs_queue_info["QueueUrl"]
resp = create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON_ECHO,
func_name=lambda_name_queue_batch,
event_source_arn=sqs_queue_info["QueueArn"],
libs=TEST_LAMBDA_LIBS,
)
event_source_id = resp["CreateEventSourceMappingResponse"]["UUID"]
lambda_client.update_event_source_mapping(UUID=event_source_id,
BatchSize=5)
messages_to_send = [{
"Id": "message{:02d}".format(i),
"MessageBody": "msgBody{:02d}".format(i),
"MessageAttributes": {
"CustomAttribute": {
"DataType": "String",
"StringValue": "CustomAttributeValue{:02d}".format(i),
}
},
} for i in range(1, 12)]
# send 11 messages (which should get split into 3 batches)
sqs_client.send_message_batch(QueueUrl=queue_url,
Entries=messages_to_send[:10])
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=messages_to_send[10]["MessageBody"],
MessageAttributes=messages_to_send[10]["MessageAttributes"],
)
def wait_for_done():
attributes = sqs_client.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=[
"ApproximateNumberOfMessages",
"ApproximateNumberOfMessagesDelayed",
"ApproximateNumberOfMessagesNotVisible",
],
)["Attributes"]
msg_count = int(attributes.get("ApproximateNumberOfMessages"))
assert 0 == msg_count, "expecting queue to be empty"
delayed_count = int(
attributes.get("ApproximateNumberOfMessagesDelayed"))
if delayed_count != 0:
LOGGER.warning(
"SQS delayed message count (actual/expected): %s/%s" %
(delayed_count, 0))
not_visible_count = int(
attributes.get("ApproximateNumberOfMessagesNotVisible"))
if not_visible_count != 0:
LOGGER.warning(
"SQS messages not visible (actual/expected): %s/%s" %
(not_visible_count, 0))
assert 0 == delayed_count, "no messages waiting for retry"
assert 0 == delayed_count + not_visible_count, "no in flight messages"
# wait for the queue to drain (max 60s)
retry(wait_for_done, retries=12, sleep=5.0)
def check_lambda_logs():
events = get_lambda_log_events(lambda_name_queue_batch, 10)
assert 3 == len(events), "expected 3 lambda invocations"
retry(check_lambda_logs, retries=5, sleep=3)
sqs_client.delete_queue(QueueUrl=queue_url)
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = "-kclapp2"
table_name = TEST_TABLE_NAME + "lsbat" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
dynamodb = aws_stack.connect_to_service("dynamodb", client=True)
dynamodb_service = aws_stack.connect_to_service("dynamodb")
dynamodbstreams = aws_stack.connect_to_service("dynamodbstreams")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
func_name=lambda_ddb_name,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId0"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId1"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId2"
},
"data": {
"S": "foobar123"
},
}
}
},
]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId3"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId4"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId5"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId0"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId1"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId2"
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId8"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId3"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId4"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId5"
}
},
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId6"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId7"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId8"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
])
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
msg = "DynamoDB updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
event_items = [
json.loads(base64.b64decode(e["data"])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get("__action_type") == "INSERT"
]
modifies = [
e for e in event_items if e.get("__action_type") == "MODIFY"
]
removes = [
e for e in event_items if e.get("__action_type") == "REMOVE"
]
self.assertEqual(num_insert, len(inserts))
self.assertEqual(num_modify, len(modifies))
self.assertEqual(num_delete, len(removes))
# assert that all inserts were received
for i, event in enumerate(inserts):
self.assertNotIn("old_image", event)
item_id = "testId%d" % i
matching = [
i for i in inserts if i["new_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["new_image"])
# assert that all updates were received
def assert_updates(expected_updates, modifies):
def found(update):
for modif in modifies:
if modif["old_image"]["id"] == update["id"]:
self.assertEqual(
modif["old_image"],
{
"id": update["id"],
"data": update["old"]
},
)
self.assertEqual(
modif["new_image"],
{
"id": update["id"],
"data": update["new"]
},
)
return True
for update in expected_updates:
self.assertTrue(found(update))
updates1 = [
{
"id": "testId6",
"old": "foobar123",
"new": "foobar123_updated1"
},
{
"id": "testId7",
"old": "foobar123",
"new": "foobar123_updated1"
},
]
updates2 = [
{
"id": "testId6",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId7",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId8",
"old": "foobar123",
"new": "foobar123_updated2"
},
]
assert_updates(updates1, modifies[:2])
assert_updates(updates2, modifies[2:])
# assert that all removes were received
for i, event in enumerate(removes):
self.assertNotIn("new_image", event)
item_id = "testId%d" % i
matching = [
i for i in removes if i["old_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["old_image"])
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=4)
# clean up
testutil.delete_lambda_function(lambda_ddb_name)
def test_put_events_with_target_sqs_event_detail_match(self):
queue_name = "queue-{}".format(short_uid())
rule_name = "rule-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
bus_name = "bus-{}".format(short_uid())
sqs_client = aws_stack.create_external_boto_client("sqs")
queue_url = sqs_client.create_queue(QueueName=queue_name)["QueueUrl"]
queue_arn = aws_stack.sqs_queue_arn(queue_name)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps({"detail": {
"EventType": ["0", "1"]
}}),
)
rs = self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[{
"Id": target_id,
"Arn": queue_arn,
"InputPath": "$.detail"
}],
)
self.assertIn("FailedEntryCount", rs)
self.assertIn("FailedEntries", rs)
self.assertEqual(0, rs["FailedEntryCount"])
self.assertEqual([], rs["FailedEntries"])
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps({"EventType": "1"}),
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp.get("Messages")
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(1, len(messages))
actual_event = json.loads(messages[0]["Body"])
self.assertEqual({"EventType": "1"}, actual_event)
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps({"EventType": "2"}),
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp.get("Messages", [])
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(0, len(messages))
# clean up
self.cleanup(bus_name, rule_name, target_id, queue_url=queue_url)
def test_put_subscription_filter_kinesis(
self,
logs_client,
logs_log_group,
logs_log_stream,
kinesis_client,
iam_client,
create_iam_role_with_policy,
):
kinesis_name = f"test-kinesis-{short_uid()}"
filter_name = "Destination"
kinesis_client.create_stream(StreamName=kinesis_name, ShardCount=1)
try:
result = kinesis_client.describe_stream(
StreamName=kinesis_name)["StreamDescription"]
kinesis_arn = result["StreamARN"]
role = f"test-kinesis-role-{short_uid()}"
policy_name = f"test-kinesis-role-policy-{short_uid()}"
role_arn = create_iam_role_with_policy(
RoleName=role,
PolicyName=policy_name,
RoleDefinition=logs_role,
PolicyDefinition=kinesis_permission,
)
# wait for stream-status "ACTIVE"
status = result["StreamStatus"]
if status != "ACTIVE":
def check_stream_active():
state = kinesis_client.describe_stream(
StreamName=kinesis_name
)["StreamDescription"]["StreamStatus"]
if state != "ACTIVE":
raise Exception(f"StreamStatus is {state}")
retry(check_stream_active,
retries=6,
sleep=1.0,
sleep_before=2.0)
def put_subscription_filter():
logs_client.put_subscription_filter(
logGroupName=logs_log_group,
filterName=filter_name,
filterPattern="",
destinationArn=kinesis_arn,
roleArn=role_arn,
)
# for a weird reason the put_subscription_filter fails on AWS the first time,
# even-though we check for ACTIVE state...
retry(put_subscription_filter, retries=6, sleep=3.0)
def put_event():
logs_client.put_log_events(
logGroupName=logs_log_group,
logStreamName=logs_log_stream,
logEvents=[
{
"timestamp": now_utc(millis=True),
"message": "test"
},
{
"timestamp": now_utc(millis=True),
"message": "test 2"
},
],
)
retry(put_event, retries=6, sleep=3.0)
shard_iterator = kinesis_client.get_shard_iterator(
StreamName=kinesis_name,
ShardId="shardId-000000000000",
ShardIteratorType="TRIM_HORIZON",
)["ShardIterator"]
response = kinesis_client.get_records(ShardIterator=shard_iterator)
# AWS sends messages as health checks
assert len(response["Records"]) >= 1
found = False
for record in response["Records"]:
data = record["Data"]
unzipped_data = gzip.decompress(data)
json_data = json.loads(unzipped_data)
if "test" in json.dumps(json_data["logEvents"]):
assert len(json_data["logEvents"]) == 2
assert json_data["logEvents"][0]["message"] == "test"
assert json_data["logEvents"][1]["message"] == "test 2"
found = True
assert found
# clean up
finally:
kinesis_client.delete_stream(StreamName=kinesis_name,
EnforceConsumerDeletion=True)
logs_client.delete_subscription_filter(logGroupName=logs_log_group,
filterName=filter_name)
def test_put_events_with_target_sns(self):
queue_name = "test-%s" % short_uid()
rule_name = "rule-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
bus_name = "bus-{}".format(short_uid())
sns_client = aws_stack.create_external_boto_client("sns")
sqs_client = aws_stack.create_external_boto_client("sqs")
topic_name = "topic-{}".format(short_uid())
topic_arn = sns_client.create_topic(Name=topic_name)["TopicArn"]
queue_url = sqs_client.create_queue(QueueName=queue_name)["QueueUrl"]
queue_arn = aws_stack.sqs_queue_arn(queue_name)
sns_client.subscribe(TopicArn=topic_arn,
Protocol="sqs",
Endpoint=queue_arn)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN),
)
rs = self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[{
"Id": target_id,
"Arn": topic_arn
}],
)
self.assertIn("FailedEntryCount", rs)
self.assertIn("FailedEntries", rs)
self.assertEqual(0, rs["FailedEntryCount"])
self.assertEqual([], rs["FailedEntries"])
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]),
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp["Messages"]
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(1, len(messages))
actual_event = json.loads(messages[0]["Body"]).get("Message")
self.assertIsValidEvent(actual_event)
self.assertEqual(TEST_EVENT_PATTERN["Detail"][0],
json.loads(actual_event).get("detail"))
# clean up
sns_client.delete_topic(TopicArn=topic_arn)
self.cleanup(bus_name, rule_name, target_id, queue_url=queue_url)
def test_set_alarm(self, sns_client, cloudwatch_client, sqs_client,
sns_create_topic, sqs_create_queue):
# create topics for state 'ALARM' and 'OK'
sns_topic_alarm = sns_create_topic()
topic_arn_alarm = sns_topic_alarm["TopicArn"]
sns_topic_ok = sns_create_topic()
topic_arn_ok = sns_topic_ok["TopicArn"]
# create queues for 'ALARM' and 'OK' (will receive sns messages)
uid = short_uid()
queue_url_alarm = sqs_create_queue(QueueName=f"AlarmQueue-{uid}")
queue_url_ok = sqs_create_queue(QueueName=f"OKQueue-{uid}")
arn_queue_alarm = sqs_client.get_queue_attributes(
QueueUrl=queue_url_alarm,
AttributeNames=["QueueArn"])["Attributes"]["QueueArn"]
arn_queue_ok = sqs_client.get_queue_attributes(
QueueUrl=queue_url_ok,
AttributeNames=["QueueArn"])["Attributes"]["QueueArn"]
sqs_client.set_queue_attributes(
QueueUrl=queue_url_alarm,
Attributes={
"Policy": get_sqs_policy(arn_queue_alarm, topic_arn_alarm)
},
)
sqs_client.set_queue_attributes(
QueueUrl=queue_url_ok,
Attributes={"Policy": get_sqs_policy(arn_queue_ok, topic_arn_ok)})
alarm_name = "test-alarm"
alarm_description = "Test Alarm when CPU exceeds 50 percent"
expected_trigger = {
"MetricName": "CPUUtilization",
"Namespace": "AWS/EC2",
"Unit": "Percent",
"Period": 300,
"EvaluationPeriods": 1,
"ComparisonOperator": "GreaterThanThreshold",
"Threshold": 50.0,
"TreatMissingData": "ignore",
"EvaluateLowSampleCountPercentile": "",
"Dimensions": [{
"value": "i-0317828c84edbe100",
"name": "InstanceId"
}],
"StatisticType": "Statistic",
"Statistic": "AVERAGE",
}
try:
# subscribe to SQS
subscription_alarm = sns_client.subscribe(TopicArn=topic_arn_alarm,
Protocol="sqs",
Endpoint=arn_queue_alarm)
subscription_ok = sns_client.subscribe(TopicArn=topic_arn_ok,
Protocol="sqs",
Endpoint=arn_queue_ok)
# create alarm with actions for "OK" and "ALARM"
cloudwatch_client.put_metric_alarm(
AlarmName=alarm_name,
AlarmDescription=alarm_description,
MetricName=expected_trigger["MetricName"],
Namespace=expected_trigger["Namespace"],
ActionsEnabled=True,
Period=expected_trigger["Period"],
Threshold=expected_trigger["Threshold"],
Dimensions=[{
"Name": "InstanceId",
"Value": "i-0317828c84edbe100"
}],
Unit=expected_trigger["Unit"],
Statistic=expected_trigger["Statistic"].capitalize(),
OKActions=[topic_arn_ok],
AlarmActions=[topic_arn_alarm],
EvaluationPeriods=expected_trigger["EvaluationPeriods"],
ComparisonOperator=expected_trigger["ComparisonOperator"],
TreatMissingData=expected_trigger["TreatMissingData"],
)
# trigger alarm
state_value = "ALARM"
state_reason = "testing alarm"
cloudwatch_client.set_alarm_state(AlarmName=alarm_name,
StateReason=state_reason,
StateValue=state_value)
retry(
check_message,
retries=PUBLICATION_RETRIES,
sleep_before=1,
sqs_client=sqs_client,
expected_queue_url=queue_url_alarm,
expected_topic_arn=topic_arn_alarm,
expected_new=state_value,
expected_reason=state_reason,
alarm_name=alarm_name,
alarm_description=alarm_description,
expected_trigger=expected_trigger,
)
# trigger OK
state_value = "OK"
state_reason = "resetting alarm"
cloudwatch_client.set_alarm_state(AlarmName=alarm_name,
StateReason=state_reason,
StateValue=state_value)
retry(
check_message,
retries=PUBLICATION_RETRIES,
sleep_before=1,
sqs_client=sqs_client,
expected_queue_url=queue_url_ok,
expected_topic_arn=topic_arn_ok,
expected_new=state_value,
expected_reason=state_reason,
alarm_name=alarm_name,
alarm_description=alarm_description,
expected_trigger=expected_trigger,
)
finally:
# cleanup
sns_client.unsubscribe(
SubscriptionArn=subscription_alarm["SubscriptionArn"])
sns_client.unsubscribe(
SubscriptionArn=subscription_ok["SubscriptionArn"])
cloudwatch_client.delete_alarms(AlarmNames=[alarm_name])
def test_scheduled_expression_events(self):
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
return 200
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
topic_name = "topic-{}".format(short_uid())
queue_name = "queue-{}".format(short_uid())
fifo_queue_name = "queue-{}.fifo".format(short_uid())
rule_name = "rule-{}".format(short_uid())
endpoint = "{}://{}:{}".format(get_service_protocol(),
config.LOCALSTACK_HOSTNAME, local_port)
sm_role_arn = aws_stack.role_arn("sfn_role")
sm_name = "state-machine-{}".format(short_uid())
topic_target_id = "target-{}".format(short_uid())
sm_target_id = "target-{}".format(short_uid())
queue_target_id = "target-{}".format(short_uid())
fifo_queue_target_id = "target-{}".format(short_uid())
events = []
state_machine_definition = """
{
"StartAt": "Hello",
"States": {
"Hello": {
"Type": "Pass",
"Result": "World",
"End": true
}
}
}
"""
state_machine_arn = self.sfn_client.create_state_machine(
name=sm_name,
definition=state_machine_definition,
roleArn=sm_role_arn)["stateMachineArn"]
topic_arn = self.sns_client.create_topic(Name=topic_name)["TopicArn"]
self.sns_client.subscribe(TopicArn=topic_arn,
Protocol="http",
Endpoint=endpoint)
queue_url = self.sqs_client.create_queue(
QueueName=queue_name)["QueueUrl"]
fifo_queue_url = self.sqs_client.create_queue(
QueueName=fifo_queue_name,
Attributes={
"FifoQueue": "true",
"ContentBasedDeduplication": "true"
},
)["QueueUrl"]
queue_arn = aws_stack.sqs_queue_arn(queue_name)
fifo_queue_arn = aws_stack.sqs_queue_arn(fifo_queue_name)
event = {"env": "testing"}
self.events_client.put_rule(Name=rule_name,
ScheduleExpression="rate(1 minutes)")
self.events_client.put_targets(
Rule=rule_name,
Targets=[
{
"Id": topic_target_id,
"Arn": topic_arn,
"Input": json.dumps(event)
},
{
"Id": sm_target_id,
"Arn": state_machine_arn,
"Input": json.dumps(event),
},
{
"Id": queue_target_id,
"Arn": queue_arn,
"Input": json.dumps(event)
},
{
"Id": fifo_queue_target_id,
"Arn": fifo_queue_arn,
"Input": json.dumps(event),
"SqsParameters": {
"MessageGroupId": "123"
},
},
],
)
def received(q_urls):
# state machine got executed
executions = self.sfn_client.list_executions(
stateMachineArn=state_machine_arn)["executions"]
self.assertGreaterEqual(len(executions), 1)
# http endpoint got events
self.assertGreaterEqual(len(events), 2)
notifications = [
event["Message"] for event in events
if event["Type"] == "Notification"
]
self.assertGreaterEqual(len(notifications), 1)
# get state machine execution detail
execution_arn = executions[0]["executionArn"]
execution_input = self.sfn_client.describe_execution(
executionArn=execution_arn)["input"]
all_msgs = []
# get message from queue
for url in q_urls:
msgs = self.sqs_client.receive_message(QueueUrl=url).get(
"Messages", [])
self.assertGreaterEqual(len(msgs), 1)
all_msgs.append(msgs[0])
return execution_input, notifications[0], all_msgs
execution_input, notification, msgs_received = retry(
received, retries=5, sleep=15, q_urls=[queue_url, fifo_queue_url])
self.assertEqual(event, json.loads(notification))
self.assertEqual(event, json.loads(execution_input))
for msg_received in msgs_received:
self.assertEqual(event, json.loads(msg_received["Body"]))
# clean up
proxy.stop()
self.cleanup(
None,
rule_name,
target_ids=[topic_target_id, sm_target_id],
queue_url=queue_url,
)
self.sns_client.delete_topic(TopicArn=topic_arn)
self.sfn_client.delete_state_machine(stateMachineArn=state_machine_arn)
def test_sqs_batch_lambda_forward(self):
sqs = aws_stack.connect_to_service('sqs')
lambda_api = aws_stack.connect_to_service('lambda')
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(
TEST_LAMBDA_NAME_QUEUE_BATCH)
queue_url = sqs_queue_info['QueueUrl']
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON_ECHO),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
resp = testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_QUEUE_BATCH,
zip_file=zip_file,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27)
event_source_id = resp['CreateEventSourceMappingResponse']['UUID']
lambda_api.update_event_source_mapping(UUID=event_source_id,
BatchSize=5)
messages_to_send = [{
'Id': 'message{:02d}'.format(i),
'MessageBody': 'msgBody{:02d}'.format(i),
'MessageAttributes': {
'CustomAttribute': {
'DataType': 'String',
'StringValue': 'CustomAttributeValue{:02d}'.format(i)
}
}
} for i in range(1, 12)]
start_time = datetime.now()
# send 11 messages (which should get split into 3 batches)
sqs.send_message_batch(QueueUrl=queue_url,
Entries=messages_to_send[:10])
sqs.send_message(
QueueUrl=queue_url,
MessageBody=messages_to_send[10]['MessageBody'],
MessageAttributes=messages_to_send[10]['MessageAttributes'])
def wait_for_done():
attributes = sqs.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=[
'ApproximateNumberOfMessages',
'ApproximateNumberOfMessagesDelayed',
'ApproximateNumberOfMessagesNotVisible'
],
)['Attributes']
msg_count = int(attributes.get('ApproximateNumberOfMessages'))
self.assertEqual(msg_count, 0, 'expecting queue to be empty')
delayed_count = int(
attributes.get('ApproximateNumberOfMessagesDelayed'))
if delayed_count != 0:
LOGGER.warning(
('SQS delayed message count (actual/expected): %s/%s') %
(delayed_count, 0))
not_visible_count = int(
attributes.get('ApproximateNumberOfMessagesNotVisible'))
if not_visible_count != 0:
LOGGER.warning(
('SQS messages not visible (actual/expected): %s/%s') %
(not_visible_count, 0))
invocation_count = get_lambda_invocations_count(
TEST_LAMBDA_NAME_QUEUE_BATCH,
period=120,
start_time=start_time,
end_time=datetime.now())
if invocation_count != 3:
LOGGER.warning(
('Lambda invocations (actual/expected): %s/%s') %
(invocation_count, 3))
self.assertEqual(delayed_count, 0, 'no messages waiting for retry')
self.assertEqual(delayed_count + not_visible_count, 0,
'no in flight messages')
self.assertEqual(invocation_count, 3,
'expected 3 lambda invocations')
# wait for the queue to drain (max 90s)
retry(wait_for_done, retries=18, sleep=5.0)
testutil.delete_lambda_function(TEST_LAMBDA_NAME_QUEUE_BATCH)
sqs.delete_queue(QueueUrl=queue_url)
def test_api_destinations(self):
token = short_uid()
bearer = "Bearer %s" % token
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
paths_list.append(path)
auth = headers.get("Api") or headers.get("Authorization")
if auth not in headers_list:
headers_list.append(auth)
return requests_response({
"access_token": token,
"token_type": "Bearer",
"expires_in": 86400,
})
events = []
paths_list = []
headers_list = []
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
events_client = aws_stack.create_external_boto_client("events")
url = "http://localhost:%s" % local_port
auth_types = [
{
"type": "BASIC",
"key": "BasicAuthParameters",
"parameters": {
"Username": "******",
"Password": "******"
},
},
{
"type": "API_KEY",
"key": "ApiKeyAuthParameters",
"parameters": {
"ApiKeyName": "Api",
"ApiKeyValue": "apikey_secret"
},
},
{
"type": "OAUTH_CLIENT_CREDENTIALS",
"key": "OAuthParameters",
"parameters": {
"AuthorizationEndpoint": url,
"ClientParameters": {
"ClientID": "id",
"ClientSecret": "password"
},
"HttpMethod": "put",
},
},
]
for auth in auth_types:
connection_name = "c-%s" % short_uid()
connection_arn = events_client.create_connection(
Name=connection_name,
AuthorizationType=auth.get("type"),
AuthParameters={
auth.get("key"): auth.get("parameters"),
"InvocationHttpParameters": {
"BodyParameters": [{
"Key": "key",
"Value": "value",
"IsValueSecret": False
}],
"HeaderParameters": [{
"Key": "key",
"Value": "value",
"IsValueSecret": False
}],
"QueryStringParameters": [{
"Key": "key",
"Value": "value",
"IsValueSecret": False
}],
},
},
)["ConnectionArn"]
# create api destination
dest_name = "d-%s" % short_uid()
result = self.events_client.create_api_destination(
Name=dest_name,
ConnectionArn=connection_arn,
InvocationEndpoint=url,
HttpMethod="POST",
)
# create rule and target
rule_name = "r-%s" % short_uid()
target_id = "target-{}".format(short_uid())
pattern = json.dumps({
"source": ["source-123"],
"detail-type": ["type-123"]
})
self.events_client.put_rule(Name=rule_name, EventPattern=pattern)
self.events_client.put_targets(
Rule=rule_name,
Targets=[{
"Id": target_id,
"Arn": result["ApiDestinationArn"]
}],
)
entries = [{
"Source": "source-123",
"DetailType": "type-123",
"Detail": '{"i": %s}' % 0,
}]
self.events_client.put_events(Entries=entries)
# cleaning
self.events_client.delete_connection(Name=connection_name)
self.events_client.delete_api_destination(Name=dest_name)
self.events_client.delete_rule(Name=rule_name, Force=True)
# assert that all events have been received in the HTTP server listener
def check():
self.assertTrue(len(events) >= len(auth_types))
self.assertTrue("key" in paths_list[0]
and "value" in paths_list[0])
self.assertTrue(events[0].get("key") == "value")
# TODO examine behavior difference between LS pro/community
# Pro seems to (correctly) use base64 for basic authentication instead of plaintext
user_pass = to_str(base64.b64encode(b"user:pass"))
self.assertTrue("Basic user:pass" in headers_list
or f"Basic {user_pass}" in headers_list)
self.assertTrue("apikey_secret" in headers_list)
self.assertTrue(bearer in headers_list)
retry(check, sleep=0.5, retries=5)
# clean up
proxy.stop()
def test_put_events_with_target_lambda(self):
rule_name = 'rule-{}'.format(short_uid())
function_name = 'lambda-func-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
rs = testutil.create_lambda_function(handler_file=os.path.join(
THIS_FOLDER, 'lambdas', 'lambda_echo.py'),
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
func_arn = rs['CreateFunctionResponse']['FunctionArn']
self.events_client.create_event_bus(Name=TEST_EVENT_BUS_NAME)
self.events_client.put_rule(
Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
EventPattern=json.dumps(TEST_EVENT_PATTERN))
rs = self.events_client.put_targets(Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Targets=[{
'Id': target_id,
'Arn': func_arn
}])
self.assertIn('FailedEntryCount', rs)
self.assertIn('FailedEntries', rs)
self.assertEqual(rs['FailedEntryCount'], 0)
self.assertEqual(rs['FailedEntries'], [])
self.events_client.put_events(
Entries=[{
'EventBusName': TEST_EVENT_BUS_NAME,
'Source': TEST_EVENT_PATTERN['Source'],
'DetailType': TEST_EVENT_PATTERN['DetailType'],
'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'])
}])
# Get lambda's log events
events = retry(check_expected_lambda_log_events_length,
retries=3,
sleep=1,
function_name=function_name,
expected_length=1)
actual_event = events[0]
self.assertIsValidEvent(actual_event)
self.assertDictEqual(json.loads(actual_event['detail']),
json.loads(TEST_EVENT_PATTERN['Detail']))
# clean up
testutil.delete_lambda_function(function_name)
self.events_client.remove_targets(Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Ids=[target_id],
Force=True)
self.events_client.delete_rule(Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Force=True)
self.events_client.delete_event_bus(Name=TEST_EVENT_BUS_NAME)
def test_put_events_with_target_kinesis(self):
rule_name = "rule-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
bus_name = "bus-{}".format(short_uid())
stream_name = "stream-{}".format(short_uid())
stream_arn = aws_stack.kinesis_stream_arn(stream_name)
kinesis_client = aws_stack.create_external_boto_client("kinesis")
kinesis_client.create_stream(StreamName=stream_name, ShardCount=1)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN),
)
put_response = self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[{
"Id": target_id,
"Arn": stream_arn,
"KinesisParameters": {
"PartitionKeyPath": "$.detail-type"
},
}],
)
self.assertIn("FailedEntryCount", put_response)
self.assertIn("FailedEntries", put_response)
self.assertEqual(0, put_response["FailedEntryCount"])
self.assertEqual([], put_response["FailedEntries"])
def check_stream_status():
_stream = kinesis_client.describe_stream(StreamName=stream_name)
assert _stream["StreamDescription"]["StreamStatus"] == "ACTIVE"
# wait until stream becomes available
retry(check_stream_status, retries=7, sleep=0.8)
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]),
}])
stream = kinesis_client.describe_stream(StreamName=stream_name)
shard_id = stream["StreamDescription"]["Shards"][0]["ShardId"]
shard_iterator = kinesis_client.get_shard_iterator(
StreamName=stream_name,
ShardId=shard_id,
ShardIteratorType="AT_TIMESTAMP",
Timestamp=datetime(2020, 1, 1),
)["ShardIterator"]
record = kinesis_client.get_records(
ShardIterator=shard_iterator)["Records"][0]
partition_key = record["PartitionKey"]
data = json.loads(record["Data"].decode())
self.assertEqual(TEST_EVENT_PATTERN["detail-type"][0], partition_key)
self.assertEqual(EVENT_DETAIL, data["detail"])
self.assertIsValidEvent(data)
def test_put_events_with_input_path_multiple(self):
queue_name = 'queue-{}'.format(short_uid())
queue_name_1 = 'queue-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
target_id_1 = 'target-{}'.format(short_uid())
bus_name = 'bus-{}'.format(short_uid())
sqs_client = aws_stack.connect_to_service('sqs')
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
queue_url_1 = sqs_client.create_queue(
QueueName=queue_name_1)['QueueUrl']
queue_arn_1 = aws_stack.sqs_queue_arn(queue_name_1)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN))
self.events_client.put_targets(Rule=rule_name,
EventBusName=bus_name,
Targets=[{
'Id': target_id,
'Arn': queue_arn,
'InputPath': '$.detail'
}, {
'Id': target_id_1,
'Arn': queue_arn_1,
}])
self.events_client.put_events(
Entries=[{
'EventBusName': bus_name,
'Source': TEST_EVENT_PATTERN['Source'][0],
'DetailType': TEST_EVENT_PATTERN['detail-type'][0],
'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0])
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp.get('Messages')
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(len(messages), 1)
self.assertEqual(json.loads(messages[0].get('Body')), EVENT_DETAIL)
messages = retry(get_message,
retries=3,
sleep=1,
queue_url=queue_url_1)
self.assertEqual(len(messages), 1)
self.assertEqual(
json.loads(messages[0].get('Body')).get('detail'), EVENT_DETAIL)
self.events_client.put_events(
Entries=[{
'EventBusName': bus_name,
'Source': 'dummySource',
'DetailType': TEST_EVENT_PATTERN['detail-type'][0],
'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0])
}])
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(messages, None)
# clean up
self.cleanup(bus_name, rule_name, target_id, queue_url=queue_url)
def test_put_events_with_input_path_multiple(self):
queue_name = "queue-{}".format(short_uid())
queue_name_1 = "queue-{}".format(short_uid())
rule_name = "rule-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
target_id_1 = "target-{}".format(short_uid())
bus_name = "bus-{}".format(short_uid())
sqs_client = aws_stack.create_external_boto_client("sqs")
queue_url = sqs_client.create_queue(QueueName=queue_name)["QueueUrl"]
queue_arn = aws_stack.sqs_queue_arn(queue_name)
queue_url_1 = sqs_client.create_queue(
QueueName=queue_name_1)["QueueUrl"]
queue_arn_1 = aws_stack.sqs_queue_arn(queue_name_1)
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN),
)
self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[
{
"Id": target_id,
"Arn": queue_arn,
"InputPath": "$.detail"
},
{
"Id": target_id_1,
"Arn": queue_arn_1,
},
],
)
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]),
}])
def get_message(queue_url):
resp = sqs_client.receive_message(QueueUrl=queue_url)
return resp.get("Messages")
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertEqual(1, len(messages))
self.assertEqual(EVENT_DETAIL, json.loads(messages[0].get("Body")))
messages = retry(get_message,
retries=3,
sleep=1,
queue_url=queue_url_1)
self.assertEqual(1, len(messages))
self.assertEqual(EVENT_DETAIL,
json.loads(messages[0].get("Body")).get("detail"))
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": "dummySource",
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]),
}])
messages = retry(get_message, retries=3, sleep=1, queue_url=queue_url)
self.assertIsNone(messages)
# clean up
self.cleanup(bus_name, rule_name, target_id, queue_url=queue_url)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
ddb_lease_table_suffix = '-kclapp'
table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=stream_name + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(stream_name, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_QUEUE,
zip_file=zip_file, event_source_arn=sqs_queue_info['QueueArn'], runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(RequestItems={table_name: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
]})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(TableName=table_name,
Key={PARTITION_KEY: {'S': 'testId%s' % i}},
AttributeUpdates={'data': {
'Action': 'PUT',
'Value': {'S': 'foobar123_updated'}
}})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIdError', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Subject='test_subject', Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
self.assertEqual(len(latest), 10)
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'], MessageBody=str(i))
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_events_kinesis
def check_events():
if len(events) != num_events:
LOGGER.warning(('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s') %
(len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [json.loads(base64.b64decode(e['data'])) for e in events]
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
self.assertEqual(len(inserts), num_put_new_items + num_batch_items)
self.assertEqual(len(modifies), num_put_existing_items + num_updates_ddb)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# check cloudwatch notifications
def check_cw_invocations():
num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM)
# TODO: It seems that CloudWatch is currently reporting an incorrect number of
# invocations, namely the sum over *all* lambdas, not the single one we're asking for.
# Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single
# Lambda invocation may happen with a set of Kinesis records, hence we cannot simply
# add num_events_ddb to num_events_lambda above!
# self.assertEqual(num_invocations, 2 + num_events_lambda)
self.assertGreater(num_invocations, num_events_sns + num_events_sqs)
num_error_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM, 'Errors')
self.assertEqual(num_error_invocations, 1)
# Lambda invocations are running asynchronously, hence sleep some time here to wait for results
retry(check_cw_invocations, retries=5, sleep=2)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM)
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
testutil.delete_lambda_function(TEST_LAMBDA_NAME_QUEUE)
sqs.delete_queue(QueueUrl=sqs_queue_info['QueueUrl'])
def test_put_events_with_target_lambda(self):
rule_name = "rule-{}".format(short_uid())
function_name = "lambda-func-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
bus_name = "bus-{}".format(short_uid())
handler_file = os.path.join(THIS_FOLDER, "lambdas", "lambda_echo.py")
rs = testutil.create_lambda_function(
handler_file=handler_file,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
func_arn = rs["CreateFunctionResponse"]["FunctionArn"]
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN),
)
rs = self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[{"Id": target_id, "Arn": func_arn}],
)
self.assertIn("FailedEntryCount", rs)
self.assertIn("FailedEntries", rs)
self.assertEqual(0, rs["FailedEntryCount"])
self.assertEqual([], rs["FailedEntries"])
self.events_client.put_events(
Entries=[
{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]),
}
]
)
# Get lambda's log events
events = retry(
check_expected_lambda_log_events_length,
retries=3,
sleep=1,
function_name=function_name,
expected_length=1,
)
actual_event = events[0]
self.assertIsValidEvent(actual_event)
self.assertDictEqual(
json.loads(actual_event["detail"]),
json.loads(TEST_EVENT_PATTERN["Detail"][0]),
)
# clean up
testutil.delete_lambda_function(function_name)
self.cleanup(bus_name, rule_name, target_id)
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
]})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {'S': 'testId%s' % i}},
AttributeUpdates={'data': {
'Action': 'PUT',
'Value': {'S': 'foobar123_updated'}
}})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
def test_put_subscription_filter_firehose(
self,
logs_client,
logs_log_group,
logs_log_stream,
s3_bucket,
s3_client,
firehose_client,
iam_client,
create_iam_role_with_policy,
):
try:
firehose_name = f"test-firehose-{short_uid()}"
s3_bucket_arn = f"arn:aws:s3:::{s3_bucket}"
role = f"test-firehose-s3-role-{short_uid()}"
policy_name = f"test-firehose-s3-role-policy-{short_uid()}"
role_arn = create_iam_role_with_policy(
RoleName=role,
PolicyName=policy_name,
RoleDefinition=s3_firehose_role,
PolicyDefinition=s3_firehose_permission,
)
# TODO AWS has troubles creating the delivery stream the first time
# policy is not accepted at first, so we try again
def create_delivery_stream():
firehose_client.create_delivery_stream(
DeliveryStreamName=firehose_name,
S3DestinationConfiguration={
"BucketARN": s3_bucket_arn,
"RoleARN": role_arn,
"BufferingHints": {
"SizeInMBs": 1,
"IntervalInSeconds": 60
},
},
)
retry(create_delivery_stream, retries=5, sleep=10.0)
response = firehose_client.describe_delivery_stream(
DeliveryStreamName=firehose_name)
firehose_arn = response["DeliveryStreamDescription"][
"DeliveryStreamARN"]
role = f"test-firehose-role-{short_uid()}"
policy_name = f"test-firehose-role-policy-{short_uid()}"
role_arn_logs = create_iam_role_with_policy(
RoleName=role,
PolicyName=policy_name,
RoleDefinition=logs_role,
PolicyDefinition=firehose_permission,
)
def check_stream_active():
state = firehose_client.describe_delivery_stream(
DeliveryStreamName=firehose_name
)["DeliveryStreamDescription"]["DeliveryStreamStatus"]
if state != "ACTIVE":
raise Exception(f"DeliveryStreamStatus is {state}")
retry(check_stream_active, retries=60, sleep=30.0)
logs_client.put_subscription_filter(
logGroupName=logs_log_group,
filterName="Destination",
filterPattern="",
destinationArn=firehose_arn,
roleArn=role_arn_logs,
)
logs_client.put_log_events(
logGroupName=logs_log_group,
logStreamName=logs_log_stream,
logEvents=[
{
"timestamp": now_utc(millis=True),
"message": "test"
},
{
"timestamp": now_utc(millis=True),
"message": "test 2"
},
],
)
def list_objects():
response = s3_client.list_objects(Bucket=s3_bucket)
assert len(response["Contents"]) >= 1
retry(list_objects, retries=60, sleep=30.0)
response = s3_client.list_objects(Bucket=s3_bucket)
key = response["Contents"][-1]["Key"]
response = s3_client.get_object(Bucket=s3_bucket, Key=key)
content = gzip.decompress(response["Body"].read()).decode("utf-8")
assert "DATA_MESSAGE" in content
assert "test" in content
assert "test 2" in content
finally:
# clean up
firehose_client.delete_delivery_stream(
DeliveryStreamName=firehose_name, AllowForceDelete=True)
def test_put_subscription_filter_lambda(
self,
lambda_client,
logs_client,
logs_log_group,
logs_log_stream,
create_lambda_function,
sts_client,
snapshot,
):
snapshot.add_transformer(snapshot.transform.lambda_api())
# special replacements for this test case:
snapshot.add_transformer(snapshot.transform.key_value("logGroupName"))
snapshot.add_transformer(snapshot.transform.key_value("logStreamName"))
snapshot.add_transformer(
KeyValueBasedTransformer(
lambda k, v:
(v if k == "eventId" and (isinstance(v, str) and re.match(
re.compile(r"^[0-9]+$"), v)) else None),
replacement="event_id",
replace_reference=False,
), )
test_lambda_name = f"test-lambda-function-{short_uid()}"
create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON3,
libs=TEST_LAMBDA_LIBS,
func_name=test_lambda_name,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
lambda_client.invoke(FunctionName=test_lambda_name, Payload=b"{}")
# get account-id to set the correct policy
account_id = sts_client.get_caller_identity()["Account"]
result = lambda_client.add_permission(
FunctionName=test_lambda_name,
StatementId=test_lambda_name,
Principal=f"logs.{config.DEFAULT_REGION}.amazonaws.com",
Action="lambda:InvokeFunction",
SourceArn=
f"arn:aws:logs:{config.DEFAULT_REGION}:{account_id}:log-group:{logs_log_group}:*",
SourceAccount=account_id,
)
snapshot.match("add_permission", result)
result = logs_client.put_subscription_filter(
logGroupName=logs_log_group,
filterName="test",
filterPattern="",
destinationArn=aws_stack.lambda_function_arn(
test_lambda_name,
account_id=account_id,
region_name=config.DEFAULT_REGION),
)
snapshot.match("put_subscription_filter", result)
logs_client.put_log_events(
logGroupName=logs_log_group,
logStreamName=logs_log_stream,
logEvents=[
{
"timestamp": now_utc(millis=True),
"message": "test"
},
{
"timestamp": now_utc(millis=True),
"message": "test 2"
},
],
)
response = logs_client.describe_subscription_filters(
logGroupName=logs_log_group)
assert len(response["subscriptionFilters"]) == 1
snapshot.match("describe_subscription_filter", response)
def check_invocation():
events = testutil.list_all_log_events(
log_group_name=logs_log_group, logs_client=logs_client)
assert len(events) == 2
events.sort(key=lambda k: k.get("message"))
snapshot.match("list_all_log_events", events)
assert isinstance(events[0]["eventId"], str)
assert "test" == events[0]["message"]
assert "test 2" in events[1]["message"]
retry(check_invocation, retries=6, sleep=3.0)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
def create_kinesis_stream(name, delete=False):
stream = aws_stack.create_kinesis_stream(name, delete=delete)
stream.wait_for()
ddb_lease_table_suffix = "-kclapp"
table_name = TEST_TABLE_NAME + "klsdss" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_stream_name = "lambda-stream-%s" % short_uid()
lambda_queue_name = "lambda-queue-%s" % short_uid()
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
queue_name = "queue-%s" % short_uid()
dynamodb = aws_stack.connect_to_resource("dynamodb")
dynamodb_service = aws_stack.connect_to_service("dynamodb")
dynamodbstreams = aws_stack.connect_to_service("dynamodbstreams")
kinesis = aws_stack.connect_to_service("kinesis")
sns = aws_stack.connect_to_service("sns")
sqs = aws_stack.connect_to_service("sqs")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
create_kinesis_stream(stream_name, delete=True)
create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS)
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# make sure we cannot create Lambda with same name twice
with self.assertRaises(Exception):
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)["StreamDescription"]["StreamARN"]
testutil.create_lambda_function(
func_name=lambda_stream_name,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(queue_name)
testutil.create_lambda_function(
func_name=lambda_queue_name,
zip_file=zip_file,
event_source_arn=sqs_queue_info["QueueArn"],
)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = (num_events_ddb - num_put_new_items -
num_put_existing_items - num_batch_items)
LOGGER.info("Putting %s items to table..." % num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: "testId%s" % i,
"data": "foobar123"
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: "testId%s" % i,
"data": "foobar123_put_existing"
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobar123 ✓"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobar123 £"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobar123 ¢"
}
}
},
]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
"S": "testId%s" % i
}},
AttributeUpdates={
"data": {
"Action": "PUT",
"Value": {
"S": "foobar123_updated"
}
}
},
)
# put items to stream
num_events_kinesis = 1
num_kinesis_records = 10
LOGGER.info("Putting %s records in %s event to stream..." %
(num_kinesis_records, num_events_kinesis))
kinesis.put_records(
Records=[{
"Data": "{}",
"PartitionKey": "testId%s" % i
} for i in range(0, num_kinesis_records)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# put 1 item to stream that will trigger an error in the Lambda
num_events_kinesis_err = 1
for i in range(num_events_kinesis_err):
kinesis.put_record(
Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey="testIdError",
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response["TopicArn"],
Protocol="lambda",
Endpoint=aws_stack.lambda_function_arn(lambda_stream_name),
)
for i in range(num_events_sns):
sns.publish(
TopicArn=response["TopicArn"],
Subject="test_subject",
Message="test message %s" % i,
)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id="shardId-000000000000",
count=10)
self.assertEqual(10, len(latest))
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info["QueueUrl"],
MessageBody=str(i))
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_kinesis_records
def check_events():
if len(events) != num_events:
msg = "DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
event_items = [
json.loads(base64.b64decode(e["data"])) for e in events
]
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [
e for e in event_items if e.get("__action_type") == "INSERT"
]
modifies = [
e for e in event_items if e.get("__action_type") == "MODIFY"
]
self.assertEqual(num_put_new_items + num_batch_items, len(inserts))
self.assertEqual(num_put_existing_items + num_updates_ddb,
len(modifies))
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=15, sleep=2)
# check cloudwatch notifications
def check_cw_invocations():
num_invocations = get_lambda_invocations_count(lambda_stream_name)
expected_invocation_count = num_events_kinesis + num_events_kinesis_err + num_events_sns
self.assertEqual(expected_invocation_count, num_invocations)
num_error_invocations = get_lambda_invocations_count(
lambda_stream_name, "Errors")
self.assertEqual(num_events_kinesis_err, num_error_invocations)
# Lambda invocations are running asynchronously, hence sleep some time here to wait for results
retry(check_cw_invocations, retries=7, sleep=2)
# clean up
testutil.delete_lambda_function(lambda_stream_name)
testutil.delete_lambda_function(lambda_ddb_name)
testutil.delete_lambda_function(lambda_queue_name)
sqs.delete_queue(QueueUrl=sqs_queue_info["QueueUrl"])
