def test_firehose_s3():
s3_resource = aws_stack.connect_to_resource('s3')
firehose = aws_stack.connect_to_service('firehose')
s3_prefix = '/testdata'
test_data = '{"test": "firehose_data_%s"}' % short_uid()
# create Firehose stream
stream = firehose.create_delivery_stream(
DeliveryStreamName=TEST_FIREHOSE_NAME,
S3DestinationConfiguration={
'RoleARN': aws_stack.iam_resource_arn('firehose'),
'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME),
'Prefix': s3_prefix
}
)
assert stream
assert TEST_FIREHOSE_NAME in firehose.list_delivery_streams()['DeliveryStreamNames']
# create target S3 bucket
s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME)
# put records
firehose.put_record(
DeliveryStreamName=TEST_FIREHOSE_NAME,
Record={
'Data': to_bytes(test_data)
}
)
# check records in target bucket
all_objects = testutil.list_all_s3_objects()
testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
def create_dynamodb_table(table_name, partition_key, env=None, stream_view_type=None):
"""Utility method to create a DynamoDB table"""
dynamodb = aws_stack.connect_to_service('dynamodb', env=env, client=True)
stream_spec = {'StreamEnabled': False}
key_schema = [{
'AttributeName': partition_key,
'KeyType': 'HASH'
}]
attr_defs = [{
'AttributeName': partition_key,
'AttributeType': 'S'
}]
if stream_view_type is not None:
stream_spec = {
'StreamEnabled': True,
'StreamViewType': stream_view_type
}
table = None
try:
table = dynamodb.create_table(TableName=table_name, KeySchema=key_schema,
AttributeDefinitions=attr_defs, ProvisionedThroughput={
'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10
},
StreamSpecification=stream_spec
)
except Exception as e:
if 'ResourceInUseException' in str(e):
# Table already exists -> return table reference
return aws_stack.connect_to_resource('dynamodb', env=env).Table(table_name)
time.sleep(2)
return table
def test_bucket_policy():
s3_resource = aws_stack.connect_to_resource('s3')
s3_client = aws_stack.connect_to_service('s3')
# create test bucket
s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME_WITH_POLICY)
# put bucket policy
policy = {
'Version': '2012-10-17',
'Statement': {
'Action': ['s3:GetObject'],
'Effect': 'Allow',
'Resource': 'arn:aws:s3:::bucketName/*',
'Principal': {
'AWS': ['*']
}
}
}
response = s3_client.put_bucket_policy(
Bucket=TEST_BUCKET_NAME_WITH_POLICY,
Policy=json.dumps(policy)
)
assert response['ResponseMetadata']['HTTPStatusCode'] == 204
# retrieve and check policy config
saved_policy = s3_client.get_bucket_policy(Bucket=TEST_BUCKET_NAME_WITH_POLICY)['Policy']
assert json.loads(saved_policy) == policy
def test_time_to_live(self):
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_client = aws_stack.connect_to_service('dynamodb')
testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME_3, partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_DDB_TABLE_NAME_3)
# Insert some items to the table
items = {
'id1': {PARTITION_KEY: 'id1', 'data': 'IT IS'},
'id2': {PARTITION_KEY: 'id2', 'data': 'TIME'},
'id3': {PARTITION_KEY: 'id3', 'data': 'TO LIVE!'}
}
for k, item in items.items():
table.put_item(Item=item)
# Describe TTL when still unset.
response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED'
# Enable TTL for given table
response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, True)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is True
# Describe TTL status after being enabled.
response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED'
# Disable TTL for given table
response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, False)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is False
# Describe TTL status after being disabled.
response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED'
# Enable TTL for given table again
response = testutil.send_update_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3, True)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveSpecification']['Enabled'] is True
# Describe TTL status after being enabled again.
response = testutil.send_describe_dynamodb_ttl_request(TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(response._content)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED'
# Clean up table
dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_3)
def test_list_stack_resources_returns_queue_urls(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_2))
cloudformation.create_stack(StackName=TEST_STACK_NAME_2, TemplateBody=template)
def check_stack():
stack = get_stack_details(TEST_STACK_NAME_2)
assert stack['StackStatus'] == 'CREATE_COMPLETE'
retry(check_stack, retries=3, sleep=2)
list_stack_summaries = list_stack_resources(TEST_STACK_NAME_2)
queue_urls = get_queue_urls()
for resource in list_stack_summaries:
assert resource['PhysicalResourceId'] in queue_urls
def get_client(resource):
resource_type = get_resource_type(resource)
service = get_service_name(resource)
resource_config = RESOURCE_TO_FUNCTION.get(resource_type)
if resource_config is None:
raise Exception('CloudFormation deployment for resource type %s not yet implemented' % resource_type)
if ACTION_CREATE not in resource_config:
# nothing to do for this resource
return
try:
if resource_config[ACTION_CREATE].get('boto_client') == 'resource':
return aws_stack.connect_to_resource(service)
return aws_stack.connect_to_service(service)
except Exception as e:
LOGGER.warning('Unable to get client for "%s" API, skipping deployment: %s' % (service, e))
return None
def test_large_data_download(self):
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_client = aws_stack.connect_to_service('dynamodb')
testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME_2, partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_DDB_TABLE_NAME_2)
# Create a large amount of items
num_items = 20
for i in range(0, num_items):
item = {PARTITION_KEY: 'id%s' % i, 'data1': 'foobar123 ' * 1000}
table.put_item(Item=item)
# Retrieve the items. The data will be transmitted to the client with chunked transfer encoding
result = table.scan(TableName=TEST_DDB_TABLE_NAME_2)
assert len(result['Items']) == num_items
# Clean up
dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_2)
def test_dynamodb_error_injection():
if not do_run():
return
dynamodb = aws_stack.connect_to_resource('dynamodb')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_TABLE_NAME)
# by default, no errors
test_no_errors = table.put_item(Item={PARTITION_KEY: short_uid(), 'data': 'foobar123'})
assert_equal(test_no_errors['ResponseMetadata']['HTTPStatusCode'], 200)
# with a probability of 1, always throw errors
config.DYNAMODB_ERROR_PROBABILITY = 1.0
assert_raises(ClientError, table.put_item, Item={PARTITION_KEY: short_uid(), 'data': 'foobar123'})
# reset probability to zero
config.DYNAMODB_ERROR_PROBABILITY = 0.0
def test_non_ascii_chars(self):
dynamodb = aws_stack.connect_to_resource('dynamodb')
testutil.create_dynamodb_table(TEST_DDB_TABLE_NAME, partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_DDB_TABLE_NAME)
# write some items containing non-ASCII characters
items = {
'id1': {PARTITION_KEY: 'id1', 'data': 'foobar123 ✓'},
'id2': {PARTITION_KEY: 'id2', 'data': 'foobar123 £'},
'id3': {PARTITION_KEY: 'id3', 'data': 'foobar123 ¢'}
}
for k, item in items.items():
table.put_item(Item=item)
for item_id in items.keys():
item = table.get_item(Key={PARTITION_KEY: item_id})['Item']
# need to fix up the JSON and convert str to unicode for Python 2
item1 = json_safe(item)
item2 = json_safe(items[item_id])
assert item1 == item2
def test_apply_template(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1))
# deploy template
cloudformation.create_stack(StackName=TEST_STACK_NAME, TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(TEST_STACK_NAME)
assert stack['StackStatus'] == 'CREATE_COMPLETE'
retry(check_stack, retries=3, sleep=2)
# assert that bucket has been created
assert bucket_exists('cf-test-bucket-1')
# assert that queue has been created
assert queue_exists('cf-test-queue-1')
# assert that stream has been created
assert stream_exists('cf-test-stream-1')
# assert that queue has been created
resource = describe_stack_resource(TEST_STACK_NAME, 'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
def invoke_rest_api(api_id,
stage,
method,
invocation_path,
data,
headers,
path=None):
path = path or invocation_path
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
# run gateway authorizers for this request
authorize_invocation(api_id, headers)
path_map = helpers.get_rest_api_paths(rest_api_id=api_id)
try:
extracted_path, resource = get_resource_for_path(path=relative_path,
path_map=path_map)
except Exception:
return make_error_response('Unable to find path %s' % path, 404)
integrations = resource.get('resourceMethods', {})
integration = integrations.get(method, {})
if not integration:
integration = integrations.get('ANY', {})
integration = integration.get('methodIntegration')
if not integration:
if method == 'OPTIONS' and 'Origin' in headers:
# default to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
return make_error_response(
'Unable to find integration for path %s' % path, 404)
uri = integration.get('uri')
if integration['type'] == 'AWS':
if 'kinesis:action/' in uri:
if uri.endswith('kinesis:action/PutRecords'):
target = kinesis_listener.ACTION_PUT_RECORDS
if uri.endswith('kinesis:action/ListStreams'):
target = kinesis_listener.ACTION_LIST_STREAMS
template = integration['requestTemplates'][APPLICATION_JSON]
new_request = aws_stack.render_velocity_template(template, data)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service='kinesis')
headers['X-Amz-Target'] = target
result = common.make_http_request(url=TEST_KINESIS_URL,
method='POST',
data=new_request,
headers=headers)
return result
if method == 'POST':
if uri.startswith('arn:aws:apigateway:') and ':sqs:path' in uri:
template = integration['requestTemplates'][APPLICATION_JSON]
account_id, queue = uri.split('/')[-2:]
region_name = uri.split(':')[3]
new_request = aws_stack.render_velocity_template(
template, data) + '&QueueName=%s' % queue
headers = aws_stack.mock_aws_request_headers(
service='sqs', region_name=region_name)
url = urljoin(TEST_SQS_URL, 'queue/%s' % queue)
result = common.make_http_request(url,
method='POST',
headers=headers,
data=new_request)
return result
msg = 'API Gateway AWS integration action URI "%s", method "%s" not yet implemented' % (
uri, method)
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration['type'] == 'AWS_PROXY':
if uri.startswith('arn:aws:apigateway:') and ':lambda:path' in uri:
func_arn = uri.split(':lambda:path')[1].split(
'functions/')[1].split('/invocations')[0]
data_str = json.dumps(data) if isinstance(data,
(dict,
list)) else to_str(data)
account_id = uri.split(':lambda:path')[1].split(
':function:')[0].split(':')[-1]
source_ip = headers['X-Forwarded-For'].split(',')[-2]
# Sample request context:
# https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html#api-gateway-create-api-as-simple-proxy-for-lambda-test
request_context = {
'path': relative_path,
'accountId': account_id,
'resourceId': resource.get('id'),
'stage': stage,
'identity': {
'accountId': account_id,
'sourceIp': source_ip,
'userAgent': headers['User-Agent'],
}
}
try:
path_params = extract_path_params(
path=relative_path, extracted_path=extracted_path)
except Exception:
path_params = {}
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
data_str,
headers,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=path,
request_context=request_context)
if isinstance(result, FlaskResponse):
return flask_to_requests_response(result)
if isinstance(result, Response):
return result
response = LambdaResponse()
parsed_result = result if isinstance(result, dict) else json.loads(
str(result))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get('statusCode', 200))
parsed_headers = parsed_result.get('headers', {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
if isinstance(parsed_result['body'], dict):
response.content = json.dumps(parsed_result['body'])
else:
response.content = to_bytes(parsed_result['body'])
except Exception:
response._content = '{}'
if response.content:
response.headers['Content-Length'] = str(len(response.content))
response.multi_value_headers = parsed_result.get(
'multiValueHeaders') or {}
return response
elif uri.startswith(
'arn:aws:apigateway:') and ':dynamodb:action' in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(':dynamodb:action')[1].split('&Table=')[1]
action = uri.split(':dynamodb:action')[1].split('&Table=')[0]
if 'PutItem' in action and method == 'PUT':
response_template = path_map.get(relative_path, {}).get('resourceMethods', {})\
.get(method, {}).get('methodIntegration', {}).\
get('integrationResponses', {}).get('200', {}).get('responseTemplates', {})\
.get('application/json', None)
if response_template is None:
msg = 'Invalid response template defined in integration response.'
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template['TableName'] != table_name:
msg = 'Invalid table name specified in integration response template.'
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource('dynamodb')
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template['Item'].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(
event_data, headers=aws_stack.mock_aws_request_headers())
return response
else:
msg = 'API Gateway action uri "%s" not yet implemented' % uri
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration['type'] == 'HTTP':
function = getattr(requests, method.lower())
if isinstance(data, dict):
data = json.dumps(data)
result = function(integration['uri'], data=data, headers=headers)
return result
else:
msg = (
'API Gateway integration type "%s" for method "%s" not yet implemented'
% (integration['type'], method))
LOGGER.warning(msg)
return make_error_response(msg, 404)
def get_dynamodb_table():
dynamodb = aws_stack.connect_to_resource('dynamodb')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY)
return dynamodb.Table(TEST_TABLE_NAME)
def invoke_rest_api_integration(api_id,
stage,
integration,
method,
path,
invocation_path,
data,
headers,
resource_path,
context={},
resource_id=None,
response_templates={}):
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
integration_type_orig = integration.get('type') or integration.get(
'integrationType') or ''
integration_type = integration_type_orig.upper()
uri = integration.get('uri') or integration.get('integrationUri')
if (uri.startswith('arn:aws:apigateway:')
and ':lambda:path' in uri) or uri.startswith('arn:aws:lambda'):
if integration_type in ['AWS', 'AWS_PROXY']:
func_arn = uri
if ':lambda:path' in uri:
func_arn = uri.split(':lambda:path')[1].split(
'functions/')[1].split('/invocations')[0]
data_str = json.dumps(data) if isinstance(data,
(dict,
list)) else to_str(data)
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
except Exception:
path_params = {}
# apply custom request template
data_str = apply_template(integration,
'request',
data_str,
path_params=path_params,
query_params=query_string_params,
headers=headers)
# Sample request context:
# https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html#api-gateway-create-api-as-simple-proxy-for-lambda-test
request_context = get_lambda_event_request_context(
method,
path,
data,
headers,
integration_uri=uri,
resource_id=resource_id)
stage_variables = get_stage_variables(api_id, stage)
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
data_str,
stage,
api_id,
headers,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=resource_path,
request_context=request_context,
event_context=context,
stage_variables=stage_variables)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = result if isinstance(
result, dict) else json.loads(str(result or '{}'))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get(
'statusCode', 200))
parsed_headers = parsed_result.get('headers', {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
if isinstance(parsed_result['body'], dict):
response._content = json.dumps(parsed_result['body'])
else:
response._content = to_bytes(parsed_result['body'])
except Exception:
response._content = '{}'
update_content_length(response)
response.multi_value_headers = parsed_result.get(
'multiValueHeaders') or {}
# apply custom response template
response._content = apply_template(integration, 'response',
response._content)
response.headers['Content-Length'] = str(
len(response.content or ''))
return response
msg = 'API Gateway %s integration action "%s", method "%s" not yet implemented' % (
integration_type, uri, method)
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration_type == 'AWS':
if 'kinesis:action/' in uri:
if uri.endswith('kinesis:action/PutRecord'):
target = kinesis_listener.ACTION_PUT_RECORD
if uri.endswith('kinesis:action/PutRecords'):
target = kinesis_listener.ACTION_PUT_RECORDS
if uri.endswith('kinesis:action/ListStreams'):
target = kinesis_listener.ACTION_LIST_STREAMS
template = integration['requestTemplates'][APPLICATION_JSON]
new_request = aws_stack.render_velocity_template(template, data)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service='kinesis')
headers['X-Amz-Target'] = target
result = common.make_http_request(url=TEST_KINESIS_URL,
method='POST',
data=new_request,
headers=headers)
# TODO apply response template..?
return result
elif 'states:action/' in uri:
if uri.endswith('states:action/StartExecution'):
action = 'StartExecution'
decoded_data = data.decode()
payload = {}
if 'stateMachineArn' in decoded_data and 'input' in decoded_data:
payload = json.loads(decoded_data)
elif APPLICATION_JSON in integration.get('requestTemplates', {}):
template = integration['requestTemplates'][APPLICATION_JSON]
payload = aws_stack.render_velocity_template(template,
data,
as_json=True)
client = aws_stack.connect_to_service('stepfunctions')
kwargs = {'name': payload['name']} if 'name' in payload else {}
result = client.start_execution(
stateMachineArn=payload['stateMachineArn'],
input=payload['input'],
**kwargs)
response = requests_response(
content={
'executionArn': result['executionArn'],
'startDate': str(result['startDate'])
},
headers=aws_stack.mock_aws_request_headers())
return response
if method == 'POST':
if uri.startswith('arn:aws:apigateway:') and ':sqs:path' in uri:
template = integration['requestTemplates'][APPLICATION_JSON]
account_id, queue = uri.split('/')[-2:]
region_name = uri.split(':')[3]
new_request = '%s&QueueName=%s' % (
aws_stack.render_velocity_template(template, data), queue)
headers = aws_stack.mock_aws_request_headers(
service='sqs', region_name=region_name)
url = urljoin(TEST_SQS_URL,
'%s/%s' % (TEST_AWS_ACCOUNT_ID, queue))
result = common.make_http_request(url,
method='POST',
headers=headers,
data=new_request)
return result
msg = 'API Gateway AWS integration action URI "%s", method "%s" not yet implemented' % (
uri, method)
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration_type == 'AWS_PROXY':
if uri.startswith('arn:aws:apigateway:') and ':dynamodb:action' in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(':dynamodb:action')[1].split('&Table=')[1]
action = uri.split(':dynamodb:action')[1].split('&Table=')[0]
if 'PutItem' in action and method == 'PUT':
response_template = response_templates.get(
'application/json', None)
if response_template is None:
msg = 'Invalid response template defined in integration response.'
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template['TableName'] != table_name:
msg = 'Invalid table name specified in integration response template.'
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource('dynamodb')
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template['Item'].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(
event_data, headers=aws_stack.mock_aws_request_headers())
return response
else:
msg = 'API Gateway action uri "%s", integration type %s not yet implemented' % (
uri, integration_type)
LOGGER.warning(msg)
return make_error_response(msg, 404)
elif integration_type in ['HTTP_PROXY', 'HTTP']:
function = getattr(requests, method.lower())
# apply custom request template
data = apply_template(integration, 'request', data)
if isinstance(data, dict):
data = json.dumps(data)
result = function(uri, data=data, headers=headers)
# apply custom response template
data = apply_template(integration, 'response', data)
return result
elif integration_type == 'MOCK':
# TODO: add logic for MOCK responses
pass
if method == 'OPTIONS':
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
msg = (
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration_type, method, uri))
LOGGER.warning(msg)
return make_error_response(msg, 404)
def test_kinesis_lambda_ddb_streams():
env = ENV_DEV
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodb_service = aws_stack.connect_to_service('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
env=env,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda (Python) connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(TEST_LAMBDA_PYTHON,
get_content=True,
libs=['localstack'],
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda (Python) connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM_PY,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
if use_docker():
# deploy test lambda (Node.js) connected to Kinesis Stream
zip_file = testutil.create_lambda_archive(
TEST_LAMBDA_NODEJS,
get_content=True,
runtime=LAMBDA_RUNTIME_NODEJS)
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_STREAM_JS,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_NODEJS)
# put items to table
num_events_ddb = 10
print('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events_ddb):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# put items to stream
num_events_kinesis = 10
print('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
print("Waiting some time before finishing test.")
time.sleep(4)
num_events = num_events_ddb + num_events_kinesis
print('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' %
(len(EVENTS), num_events))
assert len(EVENTS) == num_events
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# put items to table
num_events_ddb = 10
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events_ddb - 3):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
dynamodb.batch_write_item(
RequestItems={
TEST_TABLE_NAME: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123'
}
}
}]
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(
('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == 10
def test_kinesis_lambda_sns_ddb_sqs_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_QUEUE,
zip_file=zip_file,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
TEST_TABLE_NAME: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ✓'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 £'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ¢'
}
}
}]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {
'S': 'testId%s' % i
}},
AttributeUpdates={
'data': {
'Action': 'PUT',
'Value': {
'S': 'foobar123_updated'
}
}
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
assert len(latest) == 10
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'],
MessageBody=str(i))
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_events_kinesis
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning((
'DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s'
) % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM)
assert num_invocations == 2 + num_events_lambda
num_error_invocations = get_lambda_invocations_count(
TEST_LAMBDA_NAME_STREAM, 'Errors')
assert num_error_invocations == 1
def test_bucket_notifications():
s3_resource = aws_stack.connect_to_resource('s3')
s3_client = aws_stack.connect_to_service('s3')
sqs_client = aws_stack.connect_to_service('sqs')
# create test bucket and queue
s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
queue_info = sqs_client.create_queue(QueueName=TEST_QUEUE_NAME_FOR_S3)
# create notification on bucket
queue_url = queue_info['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(TEST_QUEUE_NAME_FOR_S3)
events = ['s3:ObjectCreated:*', 's3:ObjectRemoved:Delete']
filter_rules = {
'FilterRules': [{
'Name': 'prefix',
'Value': 'testupload/'
}, {
'Name': 'suffix',
'Value': 'testfile.txt'
}]
}
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': events,
'Filter': {
'Key': filter_rules
}
}]
}
)
# retrieve and check notification config
config = s3_client.get_bucket_notification_configuration(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
assert events == config['Events']
assert filter_rules == config['Filter']['Key']
# upload file to S3 (this should NOT trigger a notification)
test_key1 = '/testdata'
test_data1 = b'{"test": "bucket_notification1"}'
s3_client.upload_fileobj(BytesIO(test_data1), TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key1)
# upload file to S3 (this should trigger a notification)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2), TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key2)
# receive, assert, and delete message from SQS
receive_assert_delete(queue_url, [{'key': test_key2}, {'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS}], sqs_client)
# delete notification config
_delete_notification_config()
# put notification config with single event type
event = 's3:ObjectCreated:*'
s3_client.put_bucket_notification_configuration(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': [event]
}]
}
)
config = s3_client.get_bucket_notification_configuration(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
assert config['Events'] == [event]
# put notification config with single event type
event = 's3:ObjectCreated:*'
filter_rules = {
'FilterRules': [{
'Name': 'prefix',
'Value': 'testupload/'
}]
}
s3_client.put_bucket_notification_configuration(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': [event],
'Filter': {
'Key': filter_rules
}
}]
}
)
config = s3_client.get_bucket_notification_configuration(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
assert config['Events'] == [event]
assert filter_rules == config['Filter']['Key']
# upload file to S3 (this should trigger a notification)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2), TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key2)
# receive, assert, and delete message from SQS
receive_assert_delete(queue_url, [{'key': test_key2}, {'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS}], sqs_client)
# delete notification config
_delete_notification_config()
#
# Tests s3->sns->sqs notifications
#
sns_client = aws_stack.connect_to_service('sns')
topic_info = sns_client.create_topic(Name=TEST_S3_TOPIC_NAME)
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'TopicConfigurations': [
{
'Id': 'id123',
'Events': ['s3:ObjectCreated:*'],
'TopicArn': topic_info['TopicArn']
}
]
})
sns_client.subscribe(TopicArn=topic_info['TopicArn'], Protocol='sqs', Endpoint=queue_arn)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2), TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key2)
# verify subject and records
response = sqs_client.receive_message(QueueUrl=queue_url)
for message in response['Messages']:
snsObj = json.loads(message['Body'])
testutil.assert_object({'Subject': 'Amazon S3 Notification'}, snsObj)
notificationObj = json.loads(snsObj['Message'])
# notificationRecs = [ json.loads( rec_text ) for rec_text in notification
testutil.assert_objects(
[
{'key': test_key2},
{'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS}
], notificationObj['Records'])
sqs_client.delete_message(QueueUrl=queue_url, ReceiptHandle=message['ReceiptHandle'])
_delete_notification_config()
def invoke_rest_api_integration_backend(api_id,
stage,
integration,
method,
path,
invocation_path,
data,
headers,
resource_path,
context={},
resource_id=None,
response_templates={}):
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
integration_type_orig = integration.get('type') or integration.get(
'integrationType') or ''
integration_type = integration_type_orig.upper()
uri = integration.get('uri') or integration.get('integrationUri') or ''
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
except Exception:
path_params = {}
if (uri.startswith('arn:aws:apigateway:')
and ':lambda:path' in uri) or uri.startswith('arn:aws:lambda'):
if integration_type in ['AWS', 'AWS_PROXY']:
func_arn = uri
if ':lambda:path' in uri:
func_arn = uri.split(':lambda:path')[1].split(
'functions/')[1].split('/invocations')[0]
# apply custom request template
data_str = data
try:
data_str = json.dumps(data) if isinstance(
data, (dict, list)) else to_str(data)
data_str = apply_template(integration,
'request',
data_str,
path_params=path_params,
query_params=query_string_params,
headers=headers)
except Exception:
pass
# Sample request context:
# https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html#api-gateway-create-api-as-simple-proxy-for-lambda-test
request_context = get_lambda_event_request_context(
method,
path,
data,
headers,
integration_uri=uri,
resource_id=resource_id,
resource_path=resource_path)
stage_variables = get_stage_variables(api_id, stage)
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
data_str,
stage,
api_id,
headers,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=resource_path,
request_context=request_context,
event_context=context,
stage_variables=stage_variables)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = result if isinstance(
result, dict) else json.loads(str(result or '{}'))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get(
'statusCode', 200))
parsed_headers = parsed_result.get('headers', {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
if isinstance(parsed_result['body'], dict):
response._content = json.dumps(parsed_result['body'])
else:
body_bytes = to_bytes(parsed_result.get('body') or '')
if parsed_result.get('isBase64Encoded', False):
body_bytes = base64.b64decode(body_bytes)
response._content = body_bytes
except Exception as e:
LOG.warning("Couldn't set lambda response content: %s" % e)
response._content = '{}'
update_content_length(response)
response.multi_value_headers = parsed_result.get(
'multiValueHeaders') or {}
# apply custom response template
response._content = apply_template(integration, 'response',
response._content)
response.headers['Content-Length'] = str(
len(response.content or ''))
return response
raise Exception(
'API Gateway %s integration action "%s", method "%s" not yet implemented'
% (integration_type, uri, method))
elif integration_type == 'AWS':
if 'kinesis:action/' in uri:
if uri.endswith('kinesis:action/PutRecord'):
target = kinesis_listener.ACTION_PUT_RECORD
if uri.endswith('kinesis:action/PutRecords'):
target = kinesis_listener.ACTION_PUT_RECORDS
if uri.endswith('kinesis:action/ListStreams'):
target = kinesis_listener.ACTION_LIST_STREAMS
template = integration['requestTemplates'][APPLICATION_JSON]
new_request = aws_stack.render_velocity_template(template, data)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service='kinesis')
headers['X-Amz-Target'] = target
result = common.make_http_request(url=TEST_KINESIS_URL,
method='POST',
data=new_request,
headers=headers)
# TODO apply response template..?
return result
elif 'states:action/' in uri:
if uri.endswith('states:action/StartExecution'):
action = 'StartExecution'
decoded_data = data.decode()
payload = {}
if 'stateMachineArn' in decoded_data and 'input' in decoded_data:
payload = json.loads(decoded_data)
elif APPLICATION_JSON in integration.get('requestTemplates', {}):
template = integration['requestTemplates'][APPLICATION_JSON]
payload = aws_stack.render_velocity_template(template,
data,
as_json=True)
client = aws_stack.connect_to_service('stepfunctions')
kwargs = {'name': payload['name']} if 'name' in payload else {}
result = client.start_execution(
stateMachineArn=payload['stateMachineArn'],
input=payload['input'],
**kwargs)
response = requests_response(
content={
'executionArn': result['executionArn'],
'startDate': str(result['startDate'])
},
headers=aws_stack.mock_aws_request_headers())
response.headers['content-type'] = APPLICATION_JSON
return response
elif 's3:path/' in uri and method == 'GET':
s3 = aws_stack.connect_to_service('s3')
uri_match = re.match(TARGET_REGEX_S3_URI, uri)
if uri_match:
bucket, object_key = uri_match.group('bucket', 'object')
LOG.debug('Getting request for bucket %s object %s', bucket,
object_key)
try:
object = s3.get_object(Bucket=bucket, Key=object_key)
except s3.exceptions.NoSuchKey:
msg = 'Object %s not found' % object_key
LOG.debug(msg)
return make_error_response(msg, 404)
headers = aws_stack.mock_aws_request_headers(service='s3')
if object.get('ContentType'):
headers['Content-Type'] = object['ContentType']
# stream used so large files do not fill memory
response = request_response_stream(stream=object['Body'],
headers=headers)
return response
else:
msg = 'Request URI does not match s3 specifications'
LOG.warning(msg)
return make_error_response(msg, 400)
if method == 'POST':
if uri.startswith('arn:aws:apigateway:') and ':sqs:path' in uri:
template = integration['requestTemplates'][APPLICATION_JSON]
account_id, queue = uri.split('/')[-2:]
region_name = uri.split(':')[3]
new_request = '%s&QueueName=%s' % (
aws_stack.render_velocity_template(template, data), queue)
headers = aws_stack.mock_aws_request_headers(
service='sqs', region_name=region_name)
url = urljoin(TEST_SQS_URL,
'%s/%s' % (TEST_AWS_ACCOUNT_ID, queue))
result = common.make_http_request(url,
method='POST',
headers=headers,
data=new_request)
return result
raise Exception(
'API Gateway AWS integration action URI "%s", method "%s" not yet implemented'
% (uri, method))
elif integration_type == 'AWS_PROXY':
if uri.startswith('arn:aws:apigateway:') and ':dynamodb:action' in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(':dynamodb:action')[1].split('&Table=')[1]
action = uri.split(':dynamodb:action')[1].split('&Table=')[0]
if 'PutItem' in action and method == 'PUT':
response_template = response_templates.get('application/json')
if response_template is None:
msg = 'Invalid response template defined in integration response.'
LOG.info('%s Existing: %s' % (msg, response_templates))
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template['TableName'] != table_name:
msg = 'Invalid table name specified in integration response template.'
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource('dynamodb')
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template['Item'].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(
event_data, headers=aws_stack.mock_aws_request_headers())
return response
else:
raise Exception(
'API Gateway action uri "%s", integration type %s not yet implemented'
% (uri, integration_type))
elif integration_type in ['HTTP_PROXY', 'HTTP']:
if ':servicediscovery:' in uri:
# check if this is a servicediscovery integration URI
client = aws_stack.connect_to_service('servicediscovery')
service_id = uri.split('/')[-1]
instances = client.list_instances(
ServiceId=service_id)['Instances']
instance = (instances or [None])[0]
if instance and instance.get('Id'):
uri = 'http://%s/%s' % (instance['Id'],
invocation_path.lstrip('/'))
# apply custom request template
data = apply_template(integration, 'request', data)
if isinstance(data, dict):
data = json.dumps(data)
uri = apply_request_parameter(integration=integration,
path_params=path_params)
function = getattr(requests, method.lower())
result = function(uri, data=data, headers=headers)
# apply custom response template
data = apply_template(integration, 'response', data)
return result
elif integration_type == 'MOCK':
# return empty response - details filled in via responseParameters above...
return requests_response({})
if method == 'OPTIONS':
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
raise Exception(
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration_type, method, uri))
def test_create_delete_stack(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
cf_client = aws_stack.connect_to_service('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
apigateway = aws_stack.connect_to_service('apigateway')
template = template_deployer.template_to_json(
load_file(TEST_TEMPLATE_1))
# deploy template
stack_name = 'stack-%s' % short_uid()
cloudformation.create_stack(StackName=stack_name,
TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(stack_name)
self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE')
retry(check_stack, retries=3, sleep=2)
# assert that resources have been created
assert bucket_exists('cf-test-bucket-1')
assert queue_exists('cf-test-queue-1')
topic_arn = topic_exists('%s-test-topic-1-1' % stack_name)
assert topic_arn
assert stream_exists('cf-test-stream-1')
resource = describe_stack_resource(stack_name,
'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
# assert that tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(
tags, [{
'Key': 'foobar',
'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')
}])
tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags']
self.assertEqual(
tags, [{
'Key': 'foo',
'Value': 'cf-test-bucket-1'
}, {
'Key': 'bar',
'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')
}])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [
s for s in subs
if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']
]
self.assertEqual(len(subs), 1)
self.assertIn(
':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name),
subs[0]['TopicArn'])
# assert that Gateway responses have been created
test_api_name = 'test-api'
api = [
a for a in apigateway.get_rest_apis()['items']
if a['name'] == test_api_name
][0]
responses = apigateway.get_gateway_responses(
restApiId=api['id'])['items']
self.assertEqual(len(responses), 2)
types = [r['responseType'] for r in responses]
self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
# delete the stack
cf_client.delete_stack(StackName=stack_name)
# assert that resources have been deleted
assert not bucket_exists('cf-test-bucket-1')
assert not queue_exists('cf-test-queue-1')
assert not topic_exists('%s-test-topic-1-1' % stack_name)
retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def invoke_rest_api_integration_backend(
api_id,
stage,
integration,
method,
path,
invocation_path,
data,
headers,
resource_path,
context={},
resource_id=None,
response_templates={},
auth_info={},
):
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
integration_type_orig = integration.get("type") or integration.get(
"integrationType") or ""
integration_type = integration_type_orig.upper()
uri = integration.get("uri") or integration.get("integrationUri") or ""
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
except Exception:
path_params = {}
if (uri.startswith("arn:aws:apigateway:")
and ":lambda:path" in uri) or uri.startswith("arn:aws:lambda"):
if integration_type in ["AWS", "AWS_PROXY"]:
func_arn = uri
if ":lambda:path" in uri:
func_arn = (uri.split(":lambda:path")[1].split("functions/")
[1].split("/invocations")[0])
# apply custom request template
data_str = data
is_base64_encoded = False
try:
data_str = json.dumps(data) if isinstance(
data, (dict, list)) else to_str(data)
data_str = apply_template(
integration,
"request",
data_str,
path_params=path_params,
query_params=query_string_params,
headers=headers,
)
except UnicodeDecodeError:
data_str = base64.b64encode(data_str)
is_base64_encoded = True
except Exception as e:
LOG.warning(
"Unable to convert API Gateway payload to str: %s" % (e))
pass
# Sample request context:
# https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html#api-gateway-create-api-as-simple-proxy-for-lambda-test
request_context = get_lambda_event_request_context(
method,
path,
data,
headers,
integration_uri=uri,
resource_id=resource_id,
resource_path=resource_path,
auth_info=auth_info,
)
stage_variables = get_stage_variables(api_id, stage)
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
data_str,
stage,
api_id,
headers,
is_base64_encoded=is_base64_encoded,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=resource_path,
request_context=request_context,
event_context=context,
stage_variables=stage_variables,
)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = (result if isinstance(result, dict) else
json.loads(str(result or "{}")))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get(
"statusCode", 200))
parsed_headers = parsed_result.get("headers", {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
result_body = parsed_result.get("body")
if isinstance(result_body, dict):
response._content = json.dumps(result_body)
else:
body_bytes = to_bytes(to_str(result_body or ""))
if parsed_result.get("isBase64Encoded", False):
body_bytes = base64.b64decode(body_bytes)
response._content = body_bytes
except Exception as e:
LOG.warning("Couldn't set Lambda response content: %s" % e)
response._content = "{}"
update_content_length(response)
response.multi_value_headers = parsed_result.get(
"multiValueHeaders") or {}
# apply custom response template
response._content = apply_template(integration, "response",
response._content)
response.headers["Content-Length"] = str(
len(response.content or ""))
return response
raise Exception(
'API Gateway integration type "%s", action "%s", method "%s" invalid or not yet implemented'
% (integration_type, uri, method))
elif integration_type == "AWS":
if "kinesis:action/" in uri:
if uri.endswith("kinesis:action/PutRecord"):
target = kinesis_listener.ACTION_PUT_RECORD
if uri.endswith("kinesis:action/PutRecords"):
target = kinesis_listener.ACTION_PUT_RECORDS
if uri.endswith("kinesis:action/ListStreams"):
target = kinesis_listener.ACTION_LIST_STREAMS
# apply request templates
new_data = apply_request_response_templates(
data,
integration.get("requestTemplates"),
content_type=APPLICATION_JSON)
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(service="kinesis")
headers["X-Amz-Target"] = target
result = common.make_http_request(url=config.TEST_KINESIS_URL,
method="POST",
data=new_data,
headers=headers)
# apply response template
result = apply_request_response_templates(
result, response_templates, content_type=APPLICATION_JSON)
return result
elif "states:action/" in uri:
action = uri.split("/")[-1]
payload = {}
if APPLICATION_JSON in integration.get("requestTemplates", {}):
payload = apply_request_response_templates(
data,
integration.get("requestTemplates"),
content_type=APPLICATION_JSON,
as_json=True,
)
else:
payload = json.loads(data.decode("utf-8"))
client = aws_stack.connect_to_service("stepfunctions")
# Hot fix since step functions local package responses: Unsupported Operation: 'StartSyncExecution'
method_name = (camel_to_snake_case(action)
if action != "StartSyncExecution" else
"start_execution")
try:
method = getattr(client, method_name)
except AttributeError:
msg = "Invalid step function action: %s" % method_name
LOG.error(msg)
return make_error_response(msg, 400)
result = method(**payload, )
result = json_safe(
{k: result[k]
for k in result if k not in "ResponseMetadata"})
response = requests_response(
content=result,
headers=aws_stack.mock_aws_request_headers(),
)
if action == "StartSyncExecution":
# poll for the execution result and return it
result = await_sfn_execution_result(result["executionArn"])
result_status = result.get("status")
if result_status != "SUCCEEDED":
return make_error_response(
"StepFunctions execution %s failed with status '%s'" %
(result["executionArn"], result_status),
500,
)
result = json_safe(result)
response = requests_response(content=result)
# apply response templates
response = apply_request_response_templates(
response, response_templates, content_type=APPLICATION_JSON)
return response
elif "s3:path/" in uri and method == "GET":
s3 = aws_stack.connect_to_service("s3")
uri_match = re.match(TARGET_REGEX_S3_URI, uri)
if uri_match:
bucket, object_key = uri_match.group("bucket", "object")
LOG.debug("Getting request for bucket %s object %s", bucket,
object_key)
try:
object = s3.get_object(Bucket=bucket, Key=object_key)
except s3.exceptions.NoSuchKey:
msg = "Object %s not found" % object_key
LOG.debug(msg)
return make_error_response(msg, 404)
headers = aws_stack.mock_aws_request_headers(service="s3")
if object.get("ContentType"):
headers["Content-Type"] = object["ContentType"]
# stream used so large files do not fill memory
response = request_response_stream(stream=object["Body"],
headers=headers)
return response
else:
msg = "Request URI does not match s3 specifications"
LOG.warning(msg)
return make_error_response(msg, 400)
if method == "POST":
if uri.startswith("arn:aws:apigateway:") and ":sqs:path" in uri:
template = integration["requestTemplates"][APPLICATION_JSON]
account_id, queue = uri.split("/")[-2:]
region_name = uri.split(":")[3]
new_request = "%s&QueueName=%s" % (
aws_stack.render_velocity_template(template, data),
queue,
)
headers = aws_stack.mock_aws_request_headers(
service="sqs", region_name=region_name)
url = urljoin(config.TEST_SQS_URL,
"%s/%s" % (TEST_AWS_ACCOUNT_ID, queue))
result = common.make_http_request(url,
method="POST",
headers=headers,
data=new_request)
return result
raise Exception(
'API Gateway AWS integration action URI "%s", method "%s" not yet implemented'
% (uri, method))
elif integration_type == "AWS_PROXY":
if uri.startswith("arn:aws:apigateway:") and ":dynamodb:action" in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(":dynamodb:action")[1].split("&Table=")[1]
action = uri.split(":dynamodb:action")[1].split("&Table=")[0]
if "PutItem" in action and method == "PUT":
response_template = response_templates.get("application/json")
if response_template is None:
msg = "Invalid response template defined in integration response."
LOG.info("%s Existing: %s" % (msg, response_templates))
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template["TableName"] != table_name:
msg = "Invalid table name specified in integration response template."
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource("dynamodb")
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template["Item"].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(event_data)
return response
else:
raise Exception(
'API Gateway action uri "%s", integration type %s not yet implemented'
% (uri, integration_type))
elif integration_type in ["HTTP_PROXY", "HTTP"]:
if ":servicediscovery:" in uri:
# check if this is a servicediscovery integration URI
client = aws_stack.connect_to_service("servicediscovery")
service_id = uri.split("/")[-1]
instances = client.list_instances(
ServiceId=service_id)["Instances"]
instance = (instances or [None])[0]
if instance and instance.get("Id"):
uri = "http://%s/%s" % (instance["Id"],
invocation_path.lstrip("/"))
# apply custom request template
data = apply_template(integration, "request", data)
if isinstance(data, dict):
data = json.dumps(data)
uri = apply_request_parameter(integration=integration,
path_params=path_params)
function = getattr(requests, method.lower())
result = function(uri, data=data, headers=headers)
# apply custom response template
data = apply_template(integration, "response", data)
return result
elif integration_type == "MOCK":
# return empty response - details filled in via responseParameters above...
return requests_response({})
if method == "OPTIONS":
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
raise Exception(
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration_type, method, uri))
def test_bucket_notifications(self):
s3_resource = aws_stack.connect_to_resource('s3')
s3_client = aws_stack.connect_to_service('s3')
sqs_client = aws_stack.connect_to_service('sqs')
# create test bucket and queue
s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
queue_info = sqs_client.create_queue(QueueName=TEST_QUEUE_NAME_FOR_S3)
# create notification on bucket
queue_url = queue_info['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(TEST_QUEUE_NAME_FOR_S3)
events = ['s3:ObjectCreated:*', 's3:ObjectRemoved:Delete']
filter_rules = {
'FilterRules': [{
'Name': 'prefix',
'Value': 'testupload/'
}, {
'Name': 'suffix',
'Value': 'testfile.txt'
}]
}
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [
{
'Id': 'id0001',
'QueueArn': queue_arn,
'Events': events,
'Filter': {
'Key': filter_rules
}
},
{
# Add second dummy config to fix https://github.com/localstack/localstack/issues/450
'Id': 'id0002',
'QueueArn': queue_arn,
'Events': [],
'Filter': {
'Key': filter_rules
}
}
]
})
# retrieve and check notification config
config = s3_client.get_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
self.assertEqual(len(config['QueueConfigurations']), 2)
config = [c for c in config['QueueConfigurations'] if c['Events']][0]
self.assertEqual(events, config['Events'])
self.assertEqual(filter_rules, config['Filter']['Key'])
# upload file to S3 (this should NOT trigger a notification)
test_key1 = '/testdata'
test_data1 = b'{"test": "bucket_notification1"}'
s3_client.upload_fileobj(BytesIO(test_data1),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
test_key1)
# upload file to S3 (this should trigger a notification)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
test_key2)
# receive, assert, and delete message from SQS
self._receive_assert_delete(
queue_url, [{
'key': test_key2
}, {
'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS
}], sqs_client)
# delete notification config
self._delete_notification_config()
# put notification config with single event type
event = 's3:ObjectCreated:*'
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': [event]
}]
})
config = s3_client.get_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
self.assertEqual(config['Events'], [event])
# put notification config with single event type
event = 's3:ObjectCreated:*'
filter_rules = {
'FilterRules': [{
'Name': 'prefix',
'Value': 'testupload/'
}]
}
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': [event],
'Filter': {
'Key': filter_rules
}
}]
})
config = s3_client.get_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
self.assertEqual(config['Events'], [event])
self.assertEqual(filter_rules, config['Filter']['Key'])
# upload file to S3 (this should trigger a notification)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
test_key2)
# receive, assert, and delete message from SQS
self._receive_assert_delete(
queue_url, [{
'key': test_key2
}, {
'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS
}], sqs_client)
# delete notification config
self._delete_notification_config()
#
# Tests s3->sns->sqs notifications
#
sns_client = aws_stack.connect_to_service('sns')
topic_info = sns_client.create_topic(Name=TEST_S3_TOPIC_NAME)
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'TopicConfigurations': [{
'Id': 'id123',
'Events': ['s3:ObjectCreated:*'],
'TopicArn': topic_info['TopicArn']
}]
})
sns_client.subscribe(TopicArn=topic_info['TopicArn'],
Protocol='sqs',
Endpoint=queue_arn)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
test_key2)
# verify subject and records
def verify():
response = sqs_client.receive_message(QueueUrl=queue_url)
for message in response['Messages']:
snsObj = json.loads(message['Body'])
testutil.assert_object({'Subject': 'Amazon S3 Notification'},
snsObj)
notificationObj = json.loads(snsObj['Message'])
testutil.assert_objects(
[{
'key': test_key2
}, {
'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS
}], notificationObj['Records'])
sqs_client.delete_message(
QueueUrl=queue_url, ReceiptHandle=message['ReceiptHandle'])
retry(verify, retries=PUBLICATION_RETRIES, sleep=PUBLICATION_TIMEOUT)
self._delete_notification_config()
def test_bucket_notifications():
s3_resource = aws_stack.connect_to_resource('s3')
s3_client = aws_stack.connect_to_service('s3')
sqs_client = aws_stack.connect_to_service('sqs')
# create test bucket and queue
s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
queue_info = sqs_client.create_queue(QueueName=TEST_QUEUE_NAME_FOR_S3)
# create notification on bucket
queue_url = queue_info['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(TEST_QUEUE_NAME_FOR_S3)
events = ['s3:ObjectCreated:*', 's3:ObjectRemoved:Delete']
filter_rules = {
'FilterRules': [{
'Name': 'prefix',
'Value': 'testupload/'
}, {
'Name': 'suffix',
'Value': 'testfile.txt'
}]
}
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': events,
'Filter': {
'Key': filter_rules
}
}]
})
# retrieve and check notification config
config = s3_client.get_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
assert events == config['Events']
assert filter_rules == config['Filter']['Key']
# upload file to S3 (this should NOT trigger a notification)
test_key1 = '/testdata'
test_data1 = b'{"test": "bucket_notification1"}'
s3_client.upload_fileobj(BytesIO(test_data1),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key1)
# upload file to S3 (this should trigger a notification)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key2)
# receive, assert, and delete message from SQS
receive_assert_delete(queue_url,
[{
'key': test_key2
}, {
'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS
}], sqs_client)
# delete notification config
_delete_notification_config()
# put notification config with single event type
event = 's3:ObjectCreated:*'
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': [event]
}]
})
config = s3_client.get_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
assert config['Events'] == [event]
# put notification config with single event type
event = 's3:ObjectCreated:*'
filter_rules = {
'FilterRules': [{
'Name': 'prefix',
'Value': 'testupload/'
}]
}
s3_client.put_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS,
NotificationConfiguration={
'QueueConfigurations': [{
'Id': 'id123456',
'QueueArn': queue_arn,
'Events': [event],
'Filter': {
'Key': filter_rules
}
}]
})
config = s3_client.get_bucket_notification_configuration(
Bucket=TEST_BUCKET_NAME_WITH_NOTIFICATIONS)
config = config['QueueConfigurations'][0]
assert config['Events'] == [event]
assert filter_rules == config['Filter']['Key']
# upload file to S3 (this should trigger a notification)
test_key2 = 'testupload/dir1/testfile.txt'
test_data2 = b'{"test": "bucket_notification2"}'
s3_client.upload_fileobj(BytesIO(test_data2),
TEST_BUCKET_NAME_WITH_NOTIFICATIONS, test_key2)
# receive, assert, and delete message from SQS
receive_assert_delete(queue_url,
[{
'key': test_key2
}, {
'name': TEST_BUCKET_NAME_WITH_NOTIFICATIONS
}], sqs_client)
# delete notification config
_delete_notification_config()
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
]})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {'S': 'testId%s' % i}},
AttributeUpdates={'data': {
'Action': 'PUT',
'Value': {'S': 'foobar123_updated'}
}})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
def start_test(env=ENV_DEV):
try:
# setup environment
if env == ENV_DEV:
infra.start_infra(async=True)
time.sleep(6)
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams', env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating stream...')
aws_stack.create_kinesis_stream(TEST_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
env=env, stream_view_type='NEW_AND_OLD_IMAGES')
# list streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
event_source_arn = stream['StreamArn']
assert event_source_arn
# deploy test lambda
script = load_file(os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas', 'lambda_integration.py'))
zip_file = create_lambda_archive(script, get_content=True)
create_lambda_function(func_name=TEST_LAMBDA_NAME, zip_file=zip_file, event_source_arn=event_source_arn)
# put items to table
num_events = 10
print('Putting %s items to table...' % num_events)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events):
table.put_item(Item={
PARTITION_KEY: 'testId123',
'data': 'foobar123'
})
print("Waiting some time before finishing test.")
time.sleep(10)
print('DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' % (len(EVENTS), num_events))
if len(EVENTS) != num_events:
print('ERROR receiving DynamoDB updates. Running processes:')
print(run("ps aux | grep 'python\|java\|node'"))
assert len(EVENTS) == num_events
print("Test finished successfully")
cleanup(env=env)
except KeyboardInterrupt, e:
infra.KILLED = True
def setUpClass(cls):
cls.dynamodb = aws_stack.connect_to_resource('dynamodb')
def invoke_rest_api_integration_backend(
invocation_context: ApiInvocationContext):
# define local aliases from invocation context
invocation_path = invocation_context.path_with_query_string
method = invocation_context.method
data = invocation_context.data
headers = invocation_context.headers
api_id = invocation_context.api_id
stage = invocation_context.stage
resource_path = invocation_context.resource_path
response_templates = invocation_context.response_templates
integration = invocation_context.integration
# extract integration type and path parameters
relative_path, query_string_params = extract_query_string_params(
path=invocation_path)
integration_type_orig = integration.get("type") or integration.get(
"integrationType") or ""
integration_type = integration_type_orig.upper()
uri = integration.get("uri") or integration.get("integrationUri") or ""
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
invocation_context.path_params = path_params
except Exception:
path_params = {}
if (uri.startswith("arn:aws:apigateway:")
and ":lambda:path" in uri) or uri.startswith("arn:aws:lambda"):
if integration_type in ["AWS", "AWS_PROXY"]:
func_arn = uri
if ":lambda:path" in uri:
func_arn = (uri.split(":lambda:path")[1].split("functions/")
[1].split("/invocations")[0])
invocation_context.context = get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
# TODO: change this signature to InvocationContext as well!
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
payload,
stage,
api_id,
headers,
is_base64_encoded=invocation_context.is_data_base64_encoded,
path_params=path_params,
query_string_params=query_string_params,
method=method,
resource_path=resource_path,
request_context=invocation_context.context,
stage_variables=invocation_context.stage_variables,
)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = (result if isinstance(result, dict) else
json.loads(str(result or "{}")))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get(
"statusCode", 200))
parsed_headers = parsed_result.get("headers", {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
result_body = parsed_result.get("body")
if isinstance(result_body, dict):
response._content = json.dumps(result_body)
else:
body_bytes = to_bytes(to_str(result_body or ""))
if parsed_result.get("isBase64Encoded", False):
body_bytes = base64.b64decode(body_bytes)
response._content = body_bytes
except Exception as e:
LOG.warning("Couldn't set Lambda response content: %s", e)
response._content = "{}"
update_content_length(response)
response.multi_value_headers = parsed_result.get(
"multiValueHeaders") or {}
# apply custom response template
invocation_context.response = response
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
invocation_context.response.headers["Content-Length"] = str(
len(response.content or ""))
return invocation_context.response
raise Exception(
f'API Gateway integration type "{integration_type}", action "{uri}", method "{method}"'
)
elif integration_type == "AWS":
if "kinesis:action/" in uri:
if uri.endswith("kinesis:action/PutRecord"):
target = kinesis_listener.ACTION_PUT_RECORD
elif uri.endswith("kinesis:action/PutRecords"):
target = kinesis_listener.ACTION_PUT_RECORDS
elif uri.endswith("kinesis:action/ListStreams"):
target = kinesis_listener.ACTION_LIST_STREAMS
else:
LOG.info(
f"Unexpected API Gateway integration URI '{uri}' for integration type {integration_type}",
)
target = ""
try:
invocation_context.context = get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
except Exception as e:
LOG.warning("Unable to convert API Gateway payload to str", e)
raise
# forward records to target kinesis stream
headers = aws_stack.mock_aws_request_headers(
service="kinesis", region_name=invocation_context.region_name)
headers["X-Amz-Target"] = target
result = common.make_http_request(
url=config.service_url("kineses"),
data=payload,
headers=headers,
method="POST")
# apply response template
invocation_context.response = result
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
return invocation_context.response
elif "states:action/" in uri:
action = uri.split("/")[-1]
if APPLICATION_JSON in integration.get("requestTemplates", {}):
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
payload = json.loads(payload)
else:
# XXX decoding in py3 sounds wrong, this actually might break
payload = json.loads(data.decode("utf-8"))
client = aws_stack.connect_to_service("stepfunctions")
if isinstance(payload.get("input"), dict):
payload["input"] = json.dumps(payload["input"])
# Hot fix since step functions local package responses: Unsupported Operation: 'StartSyncExecution'
method_name = (camel_to_snake_case(action)
if action != "StartSyncExecution" else
"start_execution")
try:
method = getattr(client, method_name)
except AttributeError:
msg = "Invalid step function action: %s" % method_name
LOG.error(msg)
return make_error_response(msg, 400)
result = method(**payload)
result = json_safe(
{k: result[k]
for k in result if k not in "ResponseMetadata"})
response = requests_response(
content=result,
headers=aws_stack.mock_aws_request_headers(),
)
if action == "StartSyncExecution":
# poll for the execution result and return it
result = await_sfn_execution_result(result["executionArn"])
result_status = result.get("status")
if result_status != "SUCCEEDED":
return make_error_response(
"StepFunctions execution %s failed with status '%s'" %
(result["executionArn"], result_status),
500,
)
result = json_safe(result)
response = requests_response(content=result)
# apply response templates
invocation_context.response = response
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
# response = apply_request_response_templates(
# response, response_templates, content_type=APPLICATION_JSON
# )
return response
# https://docs.aws.amazon.com/apigateway/api-reference/resource/integration/
elif ("s3:path/" in uri or "s3:action/" in uri) and method == "GET":
s3 = aws_stack.connect_to_service("s3")
uri = apply_request_parameters(
uri,
integration=integration,
path_params=path_params,
query_params=query_string_params,
)
uri_match = re.match(TARGET_REGEX_PATH_S3_URI, uri) or re.match(
TARGET_REGEX_ACTION_S3_URI, uri)
if uri_match:
bucket, object_key = uri_match.group("bucket", "object")
LOG.debug("Getting request for bucket %s object %s", bucket,
object_key)
try:
object = s3.get_object(Bucket=bucket, Key=object_key)
except s3.exceptions.NoSuchKey:
msg = "Object %s not found" % object_key
LOG.debug(msg)
return make_error_response(msg, 404)
headers = aws_stack.mock_aws_request_headers(service="s3")
if object.get("ContentType"):
headers["Content-Type"] = object["ContentType"]
# stream used so large files do not fill memory
response = request_response_stream(stream=object["Body"],
headers=headers)
return response
else:
msg = "Request URI does not match s3 specifications"
LOG.warning(msg)
return make_error_response(msg, 400)
if method == "POST":
if uri.startswith("arn:aws:apigateway:") and ":sqs:path" in uri:
template = integration["requestTemplates"][APPLICATION_JSON]
account_id, queue = uri.split("/")[-2:]
region_name = uri.split(":")[3]
if "GetQueueUrl" in template or "CreateQueue" in template:
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
new_request = f"{payload}&QueueName={queue}"
else:
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
queue_url = f"{config.get_edge_url()}/{account_id}/{queue}"
new_request = f"{payload}&QueueUrl={queue_url}"
headers = aws_stack.mock_aws_request_headers(
service="sqs", region_name=region_name)
url = urljoin(config.service_url("sqs"),
f"{TEST_AWS_ACCOUNT_ID}/{queue}")
result = common.make_http_request(url,
method="POST",
headers=headers,
data=new_request)
return result
elif uri.startswith("arn:aws:apigateway:") and ":sns:path" in uri:
invocation_context.context = get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
integration_response = SnsIntegration().invoke(
invocation_context)
return apply_request_response_templates(
integration_response,
response_templates,
content_type=APPLICATION_JSON)
raise Exception(
'API Gateway AWS integration action URI "%s", method "%s" not yet implemented'
% (uri, method))
elif integration_type == "AWS_PROXY":
if uri.startswith("arn:aws:apigateway:") and ":dynamodb:action" in uri:
# arn:aws:apigateway:us-east-1:dynamodb:action/PutItem&Table=MusicCollection
table_name = uri.split(":dynamodb:action")[1].split("&Table=")[1]
action = uri.split(":dynamodb:action")[1].split("&Table=")[0]
if "PutItem" in action and method == "PUT":
response_template = response_templates.get("application/json")
if response_template is None:
msg = "Invalid response template defined in integration response."
LOG.info("%s Existing: %s", msg, response_templates)
return make_error_response(msg, 404)
response_template = json.loads(response_template)
if response_template["TableName"] != table_name:
msg = "Invalid table name specified in integration response template."
return make_error_response(msg, 404)
dynamo_client = aws_stack.connect_to_resource("dynamodb")
table = dynamo_client.Table(table_name)
event_data = {}
data_dict = json.loads(data)
for key, _ in response_template["Item"].items():
event_data[key] = data_dict[key]
table.put_item(Item=event_data)
response = requests_response(event_data)
return response
else:
raise Exception(
'API Gateway action uri "%s", integration type %s not yet implemented'
% (uri, integration_type))
elif integration_type in ["HTTP_PROXY", "HTTP"]:
if ":servicediscovery:" in uri:
# check if this is a servicediscovery integration URI
client = aws_stack.connect_to_service("servicediscovery")
service_id = uri.split("/")[-1]
instances = client.list_instances(
ServiceId=service_id)["Instances"]
instance = (instances or [None])[0]
if instance and instance.get("Id"):
uri = "http://%s/%s" % (instance["Id"],
invocation_path.lstrip("/"))
# apply custom request template
invocation_context.context = get_event_request_context(
invocation_context)
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
request_templates = RequestTemplates()
payload = request_templates.render(invocation_context)
if isinstance(payload, dict):
payload = json.dumps(payload)
uri = apply_request_parameters(uri,
integration=integration,
path_params=path_params,
query_params=query_string_params)
result = requests.request(method=method,
url=uri,
data=payload,
headers=headers)
# apply custom response template
invocation_context.response = result
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
return invocation_context.response
elif integration_type == "MOCK":
# TODO: apply tell don't ask principle inside ResponseTemplates or InvocationContext
invocation_context.stage_variables = helpers.get_stage_variables(
invocation_context)
invocation_context.response = requests_response({})
response_templates = ResponseTemplates()
response_templates.render(invocation_context)
return invocation_context.response
if method == "OPTIONS":
# fall back to returning CORS headers if this is an OPTIONS request
return get_cors_response(headers)
raise Exception(
'API Gateway integration type "%s", method "%s", URI "%s" not yet implemented'
% (integration_type, method, uri))
def test_create_delete_stack(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
cf_client = aws_stack.connect_to_service('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
apigateway = aws_stack.connect_to_service('apigateway')
template = template_deployer.template_to_json(
load_file(TEST_TEMPLATE_1))
# deploy template
stack_name = 'stack-%s' % short_uid()
cloudformation.create_stack(StackName=stack_name,
TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(stack_name)
self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE')
retry(check_stack, retries=3, sleep=2)
# assert that resources have been created
assert bucket_exists('cf-test-bucket-1')
queue_url = queue_exists('cf-test-queue-1')
assert queue_url
topic_arn = topic_exists('%s-test-topic-1-1' % stack_name)
assert topic_arn
assert stream_exists('cf-test-stream-1')
resource = describe_stack_resource(stack_name,
'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
assert ssm_param_exists('cf-test-param-1')
# assert that tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(
tags, [{
'Key': 'foobar',
'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')
}])
tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags']
self.assertEqual(
tags, [{
'Key': 'foo',
'Value': 'cf-test-bucket-1'
}, {
'Key': 'bar',
'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')
}])
queue_tags = sqs.list_queue_tags(QueueUrl=queue_url)
self.assertIn('Tags', queue_tags)
self.assertEqual(queue_tags['Tags'], {
'key1': 'value1',
'key2': 'value2'
})
# assert that bucket notifications have been created
notifications = s3.get_bucket_notification_configuration(
Bucket='cf-test-bucket-1')
self.assertIn('QueueConfigurations', notifications)
self.assertIn('LambdaFunctionConfigurations', notifications)
self.assertEqual(notifications['QueueConfigurations'][0]['QueueArn'],
'aws:arn:sqs:test:testqueue')
self.assertEqual(notifications['QueueConfigurations'][0]['Events'],
['s3:ObjectDeleted:*'])
self.assertEqual(
notifications['LambdaFunctionConfigurations'][0]
['LambdaFunctionArn'], 'aws:arn:lambda:test:testfunc')
self.assertEqual(
notifications['LambdaFunctionConfigurations'][0]['Events'],
['s3:ObjectCreated:*'])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [
s for s in subs
if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']
]
self.assertEqual(len(subs), 1)
self.assertIn(
':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name),
subs[0]['TopicArn'])
# assert that subscription attributes are added properly
attrs = sns.get_subscription_attributes(
SubscriptionArn=subs[0]['SubscriptionArn'])['Attributes']
self.assertEqual(
attrs, {
'Endpoint': subs[0]['Endpoint'],
'Protocol': 'sqs',
'SubscriptionArn': subs[0]['SubscriptionArn'],
'TopicArn': subs[0]['TopicArn'],
'FilterPolicy': json.dumps({'eventType': ['created']})
})
# assert that Gateway responses have been created
test_api_name = 'test-api'
api = [
a for a in apigateway.get_rest_apis()['items']
if a['name'] == test_api_name
][0]
responses = apigateway.get_gateway_responses(
restApiId=api['id'])['items']
self.assertEqual(len(responses), 2)
types = [r['responseType'] for r in responses]
self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
# delete the stack
cf_client.delete_stack(StackName=stack_name)
# assert that resources have been deleted
assert not bucket_exists('cf-test-bucket-1')
assert not queue_exists('cf-test-queue-1')
assert not topic_exists('%s-test-topic-1-1' % stack_name)
retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def put_records(stream_name: str, records: List[Dict]) -> Dict:
"""Put a list of records to the firehose stream - either directly from a PutRecord API call, or
received from an underlying Kinesis stream (if 'KinesisStreamAsSource' is configured)"""
stream = get_stream(stream_name)
if not stream:
return error_not_found(stream_name)
# preprocess records, add any missing attributes
add_missing_record_attributes(records)
for dest in stream.get("Destinations", []):
# apply processing steps to incoming items
proc_config = {}
for child in dest.values():
proc_config = (isinstance(child, dict)
and child.get("ProcessingConfiguration")
or proc_config)
if proc_config.get("Enabled") is not False:
for processor in proc_config.get("Processors", []):
# TODO: run processors asynchronously, to avoid request timeouts on PutRecord API calls
records = preprocess_records(processor, records)
if "ESDestinationDescription" in dest:
es_dest = dest["ESDestinationDescription"]
es_index = es_dest["IndexName"]
es_type = es_dest.get("TypeName")
es = connect_elasticsearch(endpoint=es_dest.get("ClusterEndpoint"),
domain=es_dest.get("DomainARN"))
for record in records:
obj_id = uuid.uuid4()
data = "{}"
# DirectPut
if "Data" in record:
data = base64.b64decode(record["Data"])
# KinesisAsSource
elif "data" in record:
data = base64.b64decode(record["data"])
body = json.loads(data)
try:
es.create(index=es_index,
doc_type=es_type,
id=obj_id,
body=body)
except Exception as e:
LOG.error("Unable to put record to stream: %s %s" %
(e, traceback.format_exc()))
raise e
if "S3DestinationDescription" in dest:
s3_dest = dest["S3DestinationDescription"]
bucket = s3_bucket_name(s3_dest["BucketARN"])
prefix = s3_dest.get("Prefix", "")
s3 = connect_to_resource("s3")
batched_data = b"".join([
base64.b64decode(r.get("Data") or r["data"]) for r in records
])
obj_path = get_s3_object_path(stream_name, prefix)
try:
s3.Object(bucket, obj_path).put(Body=batched_data)
except Exception as e:
LOG.error("Unable to put record to stream: %s %s" %
(e, traceback.format_exc()))
raise e
if "HttpEndpointDestinationDescription" in dest:
http_dest = dest["HttpEndpointDestinationDescription"]
end_point = http_dest["EndpointConfiguration"]
url = end_point["Url"]
record_to_send = {
"requestId": str(uuid.uuid4()),
"timestamp": (int(time.time())),
"records": [],
}
for record in records:
data = record.get("Data") or record.get("data")
record_to_send["records"].append({"data": data})
headers = {
"Content-Type": "application/json",
}
try:
requests.post(url, json=record_to_send, headers=headers)
except Exception as e:
LOG.info(
"Unable to put Firehose records to HTTP endpoint %s: %s %s"
% (url, e, traceback.format_exc()))
raise e
return {"RecordId": str(uuid.uuid4())}
def get_client(resource):
resource_type = get_resource_type(resource)
service = get_service_name(resource)
if RESOURCE_TO_FUNCTION[resource_type][ACTION_CREATE].get('boto_client') == 'resource':
return aws_stack.connect_to_resource(service)
return aws_stack.connect_to_service(service)
def start_test(env=ENV_DEV):
try:
# setup environment
if env == ENV_DEV:
infra.start_infra(async=True)
time.sleep(6)
dynamodb = aws_stack.connect_to_resource('dynamodb', env=env)
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams',
env=env)
kinesis = aws_stack.connect_to_service('kinesis', env=env)
print('Creating stream...')
aws_stack.create_kinesis_stream(TEST_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME,
listener_func=process_records,
wait_until_started=True)
print("Kinesis consumer initialized.")
# create table with stream forwarding config
create_dynamodb_table(TEST_TABLE_NAME,
partition_key=PARTITION_KEY,
env=env,
stream_view_type='NEW_AND_OLD_IMAGES')
# list streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
event_source_arn = stream['StreamArn']
assert event_source_arn
# deploy test lambda
script = load_file(
os.path.join(LOCALSTACK_ROOT_FOLDER, 'tests', 'lambdas',
'lambda_integration.py'))
zip_file = create_lambda_archive(script, get_content=True)
create_lambda_function(func_name=TEST_LAMBDA_NAME,
zip_file=zip_file,
event_source_arn=event_source_arn)
# put items to table
num_events = 10
print('Putting %s items to table...' % num_events)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_events):
table.put_item(Item={
PARTITION_KEY: 'testId123',
'data': 'foobar123'
})
print("Waiting some time before finishing test.")
time.sleep(10)
print(
'DynamoDB updates retrieved via Kinesis (actual/expected): %s/%s' %
(len(EVENTS), num_events))
if len(EVENTS) != num_events:
print('ERROR receiving DynamoDB updates. Running processes:')
print(run("ps aux | grep 'python\|java\|node'"))
assert len(EVENTS) == num_events
print("Test finished successfully")
cleanup(env=env)
except KeyboardInterrupt, e:
infra.KILLED = True
def all_s3_object_keys(bucket):
s3_client = aws_stack.connect_to_resource('s3')
bucket = s3_client.Bucket(bucket) if isinstance(bucket, str) else bucket
keys = [key for key in bucket.objects.all()]
return keys
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
def create_kinesis_stream(name, delete=False):
stream = aws_stack.create_kinesis_stream(name, delete=delete)
stream.wait_for()
ddb_lease_table_suffix = "-kclapp"
table_name = TEST_TABLE_NAME + "klsdss" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_stream_name = "lambda-stream-%s" % short_uid()
lambda_queue_name = "lambda-queue-%s" % short_uid()
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
queue_name = "queue-%s" % short_uid()
dynamodb = aws_stack.connect_to_resource("dynamodb")
dynamodb_service = aws_stack.connect_to_service("dynamodb")
dynamodbstreams = aws_stack.connect_to_service("dynamodbstreams")
kinesis = aws_stack.connect_to_service("kinesis")
sns = aws_stack.connect_to_service("sns")
sqs = aws_stack.connect_to_service("sqs")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
create_kinesis_stream(stream_name, delete=True)
create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS)
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# make sure we cannot create Lambda with same name twice
with self.assertRaises(Exception):
testutil.create_lambda_function(
func_name=lambda_ddb_name,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)["StreamDescription"]["StreamARN"]
testutil.create_lambda_function(
func_name=lambda_stream_name,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(queue_name)
testutil.create_lambda_function(
func_name=lambda_queue_name,
zip_file=zip_file,
event_source_arn=sqs_queue_info["QueueArn"],
)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = (num_events_ddb - num_put_new_items -
num_put_existing_items - num_batch_items)
LOGGER.info("Putting %s items to table..." % num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: "testId%s" % i,
"data": "foobar123"
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: "testId%s" % i,
"data": "foobar123_put_existing"
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobar123 ✓"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobar123 £"
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: short_uid(),
"data": "foobar123 ¢"
}
}
},
]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
"S": "testId%s" % i
}},
AttributeUpdates={
"data": {
"Action": "PUT",
"Value": {
"S": "foobar123_updated"
}
}
},
)
# put items to stream
num_events_kinesis = 1
num_kinesis_records = 10
LOGGER.info("Putting %s records in %s event to stream..." %
(num_kinesis_records, num_events_kinesis))
kinesis.put_records(
Records=[{
"Data": "{}",
"PartitionKey": "testId%s" % i
} for i in range(0, num_kinesis_records)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# put 1 item to stream that will trigger an error in the Lambda
num_events_kinesis_err = 1
for i in range(num_events_kinesis_err):
kinesis.put_record(
Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey="testIdError",
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME,
)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response["TopicArn"],
Protocol="lambda",
Endpoint=aws_stack.lambda_function_arn(lambda_stream_name),
)
for i in range(num_events_sns):
sns.publish(
TopicArn=response["TopicArn"],
Subject="test_subject",
Message="test message %s" % i,
)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id="shardId-000000000000",
count=10)
self.assertEqual(10, len(latest))
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info["QueueUrl"],
MessageBody=str(i))
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_kinesis_records
def check_events():
if len(events) != num_events:
msg = "DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
event_items = [
json.loads(base64.b64decode(e["data"])) for e in events
]
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [
e for e in event_items if e.get("__action_type") == "INSERT"
]
modifies = [
e for e in event_items if e.get("__action_type") == "MODIFY"
]
self.assertEqual(num_put_new_items + num_batch_items, len(inserts))
self.assertEqual(num_put_existing_items + num_updates_ddb,
len(modifies))
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=15, sleep=2)
# check cloudwatch notifications
def check_cw_invocations():
num_invocations = get_lambda_invocations_count(lambda_stream_name)
expected_invocation_count = num_events_kinesis + num_events_kinesis_err + num_events_sns
self.assertEqual(expected_invocation_count, num_invocations)
num_error_invocations = get_lambda_invocations_count(
lambda_stream_name, "Errors")
self.assertEqual(num_events_kinesis_err, num_error_invocations)
# Lambda invocations are running asynchronously, hence sleep some time here to wait for results
retry(check_cw_invocations, retries=7, sleep=2)
# clean up
testutil.delete_lambda_function(lambda_stream_name)
testutil.delete_lambda_function(lambda_ddb_name)
testutil.delete_lambda_function(lambda_queue_name)
sqs.delete_queue(QueueUrl=sqs_queue_info["QueueUrl"])
def put_records(stream_name, records):
stream = get_stream(stream_name)
if not stream:
return error_not_found(stream_name)
for dest in stream.get("Destinations", []):
if "ESDestinationDescription" in dest:
es_dest = dest["ESDestinationDescription"]
es_index = es_dest["IndexName"]
es_type = es_dest.get("TypeName")
es = connect_elasticsearch(endpoint=es_dest.get("ClusterEndpoint"),
domain=es_dest.get("DomainARN"))
for record in records:
obj_id = uuid.uuid4()
# DirectPut
if "Data" in record:
data = base64.b64decode(record["Data"])
# KinesisAsSource
elif "data" in record:
data = base64.b64decode(record["data"])
body = json.loads(data)
try:
es.create(index=es_index,
doc_type=es_type,
id=obj_id,
body=body)
except Exception as e:
LOG.error("Unable to put record to stream: %s %s" %
(e, traceback.format_exc()))
raise e
if "S3DestinationDescription" in dest:
s3_dest = dest["S3DestinationDescription"]
bucket = bucket_name(s3_dest["BucketARN"])
prefix = s3_dest.get("Prefix", "")
s3 = connect_to_resource("s3")
batched_data = b"".join([
base64.b64decode(r.get("Data") or r["data"]) for r in records
])
obj_path = get_s3_object_path(stream_name, prefix)
try:
s3.Object(bucket, obj_path).put(Body=batched_data)
except Exception as e:
LOG.error("Unable to put record to stream: %s %s" %
(e, traceback.format_exc()))
raise e
if "HttpEndpointDestinationDescription" in dest:
http_dest = dest["HttpEndpointDestinationDescription"]
end_point = http_dest["EndpointConfiguration"]
url = end_point["Url"]
record_to_send = {
"requestId": str(uuid.uuid4()),
"timestamp": (int(time.time())),
"records": [],
}
for record in records:
data = record.get("Data") or record.get("data")
record_to_send["records"].append({"data": data})
headers = {
"Content-Type": "application/json",
}
try:
requests.post(url, json=record_to_send, headers=headers)
except Exception as e:
LOG.info(
"Unable to put Firehose records to HTTP endpoint %s: %s %s"
% (url, e, traceback.format_exc()))
raise e
return {"RecordId": str(uuid.uuid4())}
def test_time_to_live(self):
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_client = aws_stack.connect_to_service('dynamodb')
aws_stack.create_dynamodb_table(TEST_DDB_TABLE_NAME_3,
partition_key=PARTITION_KEY)
table = dynamodb.Table(TEST_DDB_TABLE_NAME_3)
# Insert some items to the table
items = {
'id1': {
PARTITION_KEY: 'id1',
'data': 'IT IS'
},
'id2': {
PARTITION_KEY: 'id2',
'data': 'TIME'
},
'id3': {
PARTITION_KEY: 'id3',
'data': 'TO LIVE!'
}
}
for k, item in items.items():
table.put_item(Item=item)
# Describe TTL when still unset.
response = testutil.send_describe_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(
response._content
)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED'
# Enable TTL for given table
response = testutil.send_update_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3, True)
assert response.status_code == 200
assert json.loads(
response._content)['TimeToLiveSpecification']['Enabled'] is True
# Describe TTL status after being enabled.
response = testutil.send_describe_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(
response._content
)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED'
# Disable TTL for given table
response = testutil.send_update_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3, False)
assert response.status_code == 200
assert json.loads(
response._content)['TimeToLiveSpecification']['Enabled'] is False
# Describe TTL status after being disabled.
response = testutil.send_describe_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(
response._content
)['TimeToLiveDescription']['TimeToLiveStatus'] == 'DISABLED'
# Enable TTL for given table again
response = testutil.send_update_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3, True)
assert response.status_code == 200
assert json.loads(
response._content)['TimeToLiveSpecification']['Enabled'] is True
# Describe TTL status after being enabled again.
response = testutil.send_describe_dynamodb_ttl_request(
TEST_DDB_TABLE_NAME_3)
assert response.status_code == 200
assert json.loads(
response._content
)['TimeToLiveDescription']['TimeToLiveStatus'] == 'ENABLED'
# Clean up table
dynamodb_client.delete_table(TableName=TEST_DDB_TABLE_NAME_3)
