def process_sqs_message(message_body, queue_name):
# feed message into the first listening lambda
try:
queue_arn = aws_stack.sqs_queue_arn(queue_name)
source = next(iter(get_event_sources(source_arn=queue_arn)), None)
if source:
arn = source['FunctionArn']
event = {'Records': [{
'body': message_body,
'receiptHandle': 'MessageReceiptHandle',
'md5OfBody': md5(message_body),
'eventSourceARN': queue_arn,
'eventSource': 'aws:sqs',
'awsRegion': aws_stack.get_local_region(),
'messageId': str(uuid.uuid4()),
'attributes': {
'ApproximateFirstReceiveTimestamp': '{}000'.format(int(time.time())),
'SenderId': '123456789012',
'ApproximateReceiveCount': '1',
'SentTimestamp': '{}000'.format(int(time.time()))
},
'messageAttributes': {},
'sqs': True,
}]}
run_lambda(event=event, context={}, func_arn=arn)
return True
except Exception as e:
LOG.warning('Unable to run Lambda function on SQS messages: %s %s' % (e, traceback.format_exc()))
def test_stream_spec_and_region_replacement(self):
aws_stack.create_dynamodb_table(TEST_DDB_TABLE_NAME_4,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
table = self.dynamodb.Table(TEST_DDB_TABLE_NAME_4)
# assert ARN formats
expected_arn_prefix = 'arn:aws:dynamodb:' + aws_stack.get_local_region(
)
self.assertTrue(table.table_arn.startswith(expected_arn_prefix))
self.assertTrue(
table.latest_stream_arn.startswith(expected_arn_prefix))
# assert shard ID formats
ddbstreams = aws_stack.connect_to_service('dynamodbstreams')
result = ddbstreams.describe_stream(
StreamArn=table.latest_stream_arn)['StreamDescription']
self.assertIn('Shards', result)
for shard in result['Shards']:
self.assertRegex(shard['ShardId'],
r'^shardId\-[0-9]{20}\-[a-zA-Z0-9]{1,36}$')
# clean up
delete_table(TEST_DDB_TABLE_NAME_4)
def test_stream_spec_and_region_replacement(self):
ddbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
table_name = 'ddb-%s' % short_uid()
aws_stack.create_dynamodb_table(
table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES')
table = self.dynamodb.Table(table_name)
# assert ARN formats
expected_arn_prefix = 'arn:aws:dynamodb:' + aws_stack.get_local_region()
self.assertTrue(table.table_arn.startswith(expected_arn_prefix))
self.assertTrue(table.latest_stream_arn.startswith(expected_arn_prefix))
# assert stream has been created
stream_tables = [s['TableName'] for s in ddbstreams.list_streams()['Streams']]
self.assertIn(table_name, stream_tables)
stream_name = get_kinesis_stream_name(table_name)
self.assertIn(stream_name, kinesis.list_streams()['StreamNames'])
# assert shard ID formats
result = ddbstreams.describe_stream(StreamArn=table.latest_stream_arn)['StreamDescription']
self.assertIn('Shards', result)
for shard in result['Shards']:
self.assertRegex(shard['ShardId'], r'^shardId\-[0-9]{20}\-[a-zA-Z0-9]{1,36}$')
# clean up
delete_table(table_name)
# assert stream has been deleted
stream_tables = [s['TableName'] for s in ddbstreams.list_streams()['Streams']]
self.assertNotIn(table_name, stream_tables)
self.assertNotIn(stream_name, kinesis.list_streams()['StreamNames'])
def extract_region_from_auth_header(headers):
# TODO: use method from aws_stack directly (leaving import here for now, to avoid circular dependency)
from localstack.utils.aws import aws_stack
auth = headers.get("Authorization") or ""
region = re.sub(r".*Credential=[^/]+/[^/]+/([^/]+)/.*", r"\1", auth)
if region == auth:
return None
region = region or aws_stack.get_local_region()
return region
def test_region_replacement(self):
dynamodb = aws_stack.connect_to_resource('dynamodb')
aws_stack.create_dynamodb_table(TEST_DDB_TABLE_NAME_4,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
table = dynamodb.Table(TEST_DDB_TABLE_NAME_4)
expected_arn_prefix = 'arn:aws:dynamodb:' + aws_stack.get_local_region(
)
assert table.table_arn.startswith(expected_arn_prefix)
assert table.latest_stream_arn.startswith(expected_arn_prefix)
def extract_region_from_headers(headers):
region = headers.get(MARKER_APIGW_REQUEST_REGION)
# Fix region lookup for certain requests, e.g., API gateway invocations
# that do not contain region details in the Authorization header.
if region:
return region
region = extract_region_from_auth_header(headers)
if not region:
# fall back to local region
region = aws_stack.get_local_region()
return region
def test_region_replacement(self):
aws_stack.create_dynamodb_table(
TEST_DDB_TABLE_NAME_4,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES'
)
table = self.dynamodb.Table(TEST_DDB_TABLE_NAME_4)
expected_arn_prefix = 'arn:aws:dynamodb:' + aws_stack.get_local_region()
self.assertTrue(table.table_arn.startswith(expected_arn_prefix))
self.assertTrue(table.latest_stream_arn.startswith(expected_arn_prefix))
# clean up
delete_table(TEST_DDB_TABLE_NAME_4)
def test_stream_spec_and_region_replacement(self):
ddbstreams = aws_stack.connect_to_service("dynamodbstreams")
kinesis = aws_stack.connect_to_service("kinesis")
table_name = "ddb-%s" % short_uid()
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
table = self.dynamodb.Table(table_name)
# assert ARN formats
expected_arn_prefix = "arn:aws:dynamodb:" + aws_stack.get_local_region(
)
self.assertTrue(table.table_arn.startswith(expected_arn_prefix))
self.assertTrue(
table.latest_stream_arn.startswith(expected_arn_prefix))
# assert stream has been created
stream_tables = [
s["TableName"] for s in ddbstreams.list_streams()["Streams"]
]
self.assertIn(table_name, stream_tables)
stream_name = get_kinesis_stream_name(table_name)
self.assertIn(stream_name, kinesis.list_streams()["StreamNames"])
# assert shard ID formats
result = ddbstreams.describe_stream(
StreamArn=table.latest_stream_arn)["StreamDescription"]
self.assertIn("Shards", result)
for shard in result["Shards"]:
self.assertRegex(shard["ShardId"],
r"^shardId\-[0-9]{20}\-[a-zA-Z0-9]{1,36}$")
# clean up
delete_table(table_name)
# assert stream has been deleted
stream_tables = [
s["TableName"] for s in ddbstreams.list_streams()["Streams"]
]
self.assertNotIn(table_name, stream_tables)
self.assertNotIn(stream_name, kinesis.list_streams()["StreamNames"])
def test_stream_spec_and_region_replacement(self, dynamodb):
ddbstreams = aws_stack.create_external_boto_client("dynamodbstreams")
kinesis = aws_stack.create_external_boto_client("kinesis")
table_name = "ddb-%s" % short_uid()
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
table = dynamodb.Table(table_name)
# assert ARN formats
expected_arn_prefix = "arn:aws:dynamodb:" + aws_stack.get_local_region(
)
assert table.table_arn.startswith(expected_arn_prefix)
assert table.latest_stream_arn.startswith(expected_arn_prefix)
# assert stream has been created
stream_tables = [
s["TableName"] for s in ddbstreams.list_streams()["Streams"]
]
assert table_name in stream_tables
stream_name = get_kinesis_stream_name(table_name)
assert stream_name in kinesis.list_streams()["StreamNames"]
# assert shard ID formats
result = ddbstreams.describe_stream(
StreamArn=table.latest_stream_arn)["StreamDescription"]
assert "Shards" in result
for shard in result["Shards"]:
assert re.match(r"^shardId-[0-9]{20}-[a-zA-Z0-9]{1,36}$",
shard["ShardId"])
# clean up
delete_table(table_name)
# assert stream has been deleted
stream_tables = [
s["TableName"] for s in ddbstreams.list_streams()["Streams"]
]
assert table_name not in stream_tables
assert stream_name not in kinesis.list_streams()["StreamNames"]
def return_response(self, method, path, data, headers, response):
data = json.loads(to_str(data))
# update table definitions
if data and 'TableName' in data and 'KeySchema' in data:
TABLE_DEFINITIONS[data['TableName']] = data
if response._content:
# fix the table ARN (DynamoDBLocal hardcodes "ddblocal" as the region)
content_replaced = re.sub(
r'"TableArn"\s*:\s*"arn:aws:dynamodb:ddblocal:([^"]+)"',
r'"TableArn": "arn:aws:dynamodb:%s:\1"' %
aws_stack.get_local_region(), to_str(response._content))
if content_replaced != response._content:
response._content = content_replaced
fix_headers_for_updated_response(response)
action = headers.get('X-Amz-Target')
if not action:
return
record = {
'eventID': '1',
'eventVersion': '1.0',
'dynamodb': {
'StreamViewType': 'NEW_AND_OLD_IMAGES',
'SizeBytes': -1
},
'awsRegion': DEFAULT_REGION,
'eventSource': 'aws:dynamodb'
}
records = [record]
if action == '%s.UpdateItem' % ACTION_PREFIX:
updated_item = find_existing_item(data)
if not updated_item:
return
record['eventName'] = 'MODIFY'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb'][
'OldImage'] = ProxyListenerDynamoDB.thread_local.existing_item
record['dynamodb']['NewImage'] = updated_item
record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
records = []
for table_name, requests in data['RequestItems'].items():
for request in requests:
put_request = request.get('PutRequest')
if put_request:
keys = dynamodb_extract_keys(item=put_request['Item'],
table_name=table_name)
if isinstance(keys, Response):
return keys
new_record = clone(record)
new_record['eventName'] = 'INSERT'
new_record['dynamodb']['Keys'] = keys
new_record['dynamodb']['NewImage'] = put_request[
'Item']
new_record[
'eventSourceARN'] = aws_stack.dynamodb_table_arn(
table_name)
records.append(new_record)
elif action == '%s.PutItem' % ACTION_PREFIX:
existing_item = ProxyListenerDynamoDB.thread_local.existing_item
ProxyListenerDynamoDB.thread_local.existing_item = None
record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
keys = dynamodb_extract_keys(item=data['Item'],
table_name=data['TableName'])
if isinstance(keys, Response):
return keys
record['dynamodb']['Keys'] = keys
record['dynamodb']['NewImage'] = data['Item']
record['dynamodb']['SizeBytes'] = len(json.dumps(data['Item']))
elif action == '%s.GetItem' % ACTION_PREFIX:
if response.status_code == 200:
content = json.loads(to_str(response.content))
# make sure we append 'ConsumedCapacity', which is properly
# returned by dynalite, but not by AWS's DynamoDBLocal
if 'ConsumedCapacity' not in content and data.get(
'ReturnConsumedCapacity') in ('TOTAL', 'INDEXES'):
content['ConsumedCapacity'] = {
'CapacityUnits': 0.5, # TODO hardcoded
'TableName': data['TableName']
}
response._content = json.dumps(content)
fix_headers_for_updated_response(response)
elif action == '%s.DeleteItem' % ACTION_PREFIX:
record['eventName'] = 'REMOVE'
record['dynamodb']['Keys'] = data['Key']
elif action == '%s.CreateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_CREATE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.DeleteTable' % ACTION_PREFIX:
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_DELETE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.UpdateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
return
else:
# nothing to do
return
if len(records) > 0 and 'eventName' in records[0]:
if 'TableName' in data:
records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn(
data['TableName'])
forward_to_lambda(records)
forward_to_ddb_stream(records)
def return_response(self, method, path, data, headers, response):
if path.startswith('/shell'):
return
data = json.loads(to_str(data))
# update table definitions
if data and 'TableName' in data and 'KeySchema' in data:
TABLE_DEFINITIONS[data['TableName']] = data
if response._content:
# fix the table and latest stream ARNs (DynamoDBLocal hardcodes "ddblocal" as the region)
content_replaced = re.sub(
r'("TableArn"|"LatestStreamArn"|"StreamArn")\s*:\s*"arn:aws:dynamodb:'
+ 'ddblocal:([^"]+)"',
r'\1: "arn:aws:dynamodb:%s:\2"' % aws_stack.get_local_region(),
to_str(response._content))
if content_replaced != response._content:
response._content = content_replaced
fix_headers_for_updated_response(response)
action = headers.get('X-Amz-Target')
if not action:
return
record = {
'eventID': '1',
'eventVersion': '1.0',
'dynamodb': {
'StreamViewType': 'NEW_AND_OLD_IMAGES',
'SizeBytes': -1
},
'awsRegion': config.DEFAULT_REGION,
'eventSource': 'aws:dynamodb'
}
records = [record]
if action == '%s.UpdateItem' % ACTION_PREFIX:
if response.status_code == 200:
updated_item = find_existing_item(data)
if not updated_item:
return
record['eventName'] = 'MODIFY'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = self._thread_local(
'existing_item')
record['dynamodb']['NewImage'] = updated_item
record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
records = self.prepare_batch_write_item_records(record, data)
elif action == '%s.TransactWriteItems' % ACTION_PREFIX:
records = self.prepare_transact_write_item_records(record, data)
elif action == '%s.PutItem' % ACTION_PREFIX:
if response.status_code == 200:
existing_item = self._thread_local('existing_item')
record[
'eventName'] = 'INSERT' if not existing_item else 'MODIFY'
keys = dynamodb_extract_keys(item=data['Item'],
table_name=data['TableName'])
if isinstance(keys, Response):
return keys
record['dynamodb']['Keys'] = keys
record['dynamodb']['NewImage'] = data['Item']
record['dynamodb']['SizeBytes'] = len(json.dumps(data['Item']))
if existing_item:
record['dynamodb']['OldImage'] = existing_item
elif action == '%s.GetItem' % ACTION_PREFIX:
if response.status_code == 200:
content = json.loads(to_str(response.content))
# make sure we append 'ConsumedCapacity', which is properly
# returned by dynalite, but not by AWS's DynamoDBLocal
if 'ConsumedCapacity' not in content and data.get(
'ReturnConsumedCapacity') in ('TOTAL', 'INDEXES'):
content['ConsumedCapacity'] = {
'CapacityUnits': 0.5, # TODO hardcoded
'TableName': data['TableName']
}
response._content = json.dumps(content)
fix_headers_for_updated_response(response)
elif action == '%s.DeleteItem' % ACTION_PREFIX:
if response.status_code == 200:
old_item = self._thread_local('existing_item')
record['eventName'] = 'REMOVE'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = old_item
elif action == '%s.CreateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
if response.status_code == 200:
content = json.loads(to_str(response._content))
create_dynamodb_stream(
data, content['TableDescription']['LatestStreamLabel'])
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_CREATE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.DeleteTable' % ACTION_PREFIX:
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_DELETE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.UpdateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
if response.status_code == 200:
content = json.loads(to_str(response._content))
create_dynamodb_stream(
data, content['TableDescription']['LatestStreamLabel'])
return
else:
# nothing to do
return
if len(records) > 0 and 'eventName' in records[0]:
if 'TableName' in data:
records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn(
data['TableName'])
forward_to_lambda(records)
forward_to_ddb_stream(records)
def return_response(self, method, path, data, headers, response):
data = json.loads(to_str(data))
# update table definitions
if data and 'TableName' in data and 'KeySchema' in data:
TABLE_DEFINITIONS[data['TableName']] = data
if response._content:
# fix the table ARN (DynamoDBLocal hardcodes "ddblocal" as the region)
content_replaced = re.sub(r'"TableArn"\s*:\s*"arn:aws:dynamodb:ddblocal:([^"]+)"',
r'"TableArn": "arn:aws:dynamodb:%s:\1"' % aws_stack.get_local_region(), to_str(response._content))
if content_replaced != response._content:
response._content = content_replaced
fix_headers_for_updated_response(response)
action = headers.get('X-Amz-Target')
if not action:
return
record = {
'eventID': '1',
'eventVersion': '1.0',
'dynamodb': {
'StreamViewType': 'NEW_AND_OLD_IMAGES',
'SizeBytes': -1
},
'awsRegion': DEFAULT_REGION,
'eventSource': 'aws:dynamodb'
}
records = [record]
if action == '%s.UpdateItem' % ACTION_PREFIX:
updated_item = find_existing_item(data)
if not updated_item:
return
record['eventName'] = 'MODIFY'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = ProxyListenerDynamoDB.thread_local.existing_item
record['dynamodb']['NewImage'] = updated_item
record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
records = []
for table_name, requests in data['RequestItems'].items():
for request in requests:
put_request = request.get('PutRequest')
if put_request:
keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name)
if isinstance(keys, Response):
return keys
new_record = clone(record)
new_record['eventName'] = 'INSERT'
new_record['dynamodb']['Keys'] = keys
new_record['dynamodb']['NewImage'] = put_request['Item']
new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn(table_name)
records.append(new_record)
elif action == '%s.PutItem' % ACTION_PREFIX:
existing_item = ProxyListenerDynamoDB.thread_local.existing_item
ProxyListenerDynamoDB.thread_local.existing_item = None
record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
keys = dynamodb_extract_keys(item=data['Item'], table_name=data['TableName'])
if isinstance(keys, Response):
return keys
record['dynamodb']['Keys'] = keys
record['dynamodb']['NewImage'] = data['Item']
record['dynamodb']['SizeBytes'] = len(json.dumps(data['Item']))
elif action == '%s.GetItem' % ACTION_PREFIX:
if response.status_code == 200:
content = json.loads(to_str(response.content))
# make sure we append 'ConsumedCapacity', which is properly
# returned by dynalite, but not by AWS's DynamoDBLocal
if 'ConsumedCapacity' not in content and data.get('ReturnConsumedCapacity') in ('TOTAL', 'INDEXES'):
content['ConsumedCapacity'] = {
'CapacityUnits': 0.5, # TODO hardcoded
'TableName': data['TableName']
}
response._content = json.dumps(content)
fix_headers_for_updated_response(response)
elif action == '%s.DeleteItem' % ACTION_PREFIX:
old_item = ProxyListenerDynamoDB.thread_local.existing_item
record['eventName'] = 'REMOVE'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = old_item
elif action == '%s.CreateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
event_publisher.fire_event(event_publisher.EVENT_DYNAMODB_CREATE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.DeleteTable' % ACTION_PREFIX:
event_publisher.fire_event(event_publisher.EVENT_DYNAMODB_DELETE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.UpdateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
return
else:
# nothing to do
return
if len(records) > 0 and 'eventName' in records[0]:
if 'TableName' in data:
records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn(data['TableName'])
forward_to_lambda(records)
forward_to_ddb_stream(records)
