def test_upload_lambda_from_s3():
s3_client = aws_stack.connect_to_service('s3')
lambda_client = aws_stack.connect_to_service('lambda')
lambda_name = 'test_lambda_%s' % short_uid()
bucket_name = 'test_bucket_lambda'
bucket_key = 'test_lambda.zip'
# upload zip file to S3
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
s3_client.create_bucket(Bucket=bucket_name)
s3_client.upload_fileobj(BytesIO(zip_file), bucket_name, bucket_key)
# create lambda function
lambda_client.create_function(
FunctionName=lambda_name, Handler='handler.handler',
Runtime=lambda_api.LAMBDA_RUNTIME_PYTHON27, Role='r1',
Code={
'S3Bucket': bucket_name,
'S3Key': bucket_key
}
)
# invoke lambda function
data_before = b'{"foo": "bar"}'
result = lambda_client.invoke(FunctionName=lambda_name, Payload=data_before)
data_after = result['Payload'].read()
assert json.loads(to_str(data_before)) == json.loads(to_str(data_after))
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(to_str(data))
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM if action == ACTION_CREATE_STREAM
else event_publisher.EVENT_KINESIS_DELETE_STREAM)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(data.get('StreamName'))})
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber': response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
def return_response(self, method, path, data, headers, response):
req_data = None
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action')[0]
if req_data:
if action == 'DescribeStackResources':
if response.status_code < 300:
response_dict = xmltodict.parse(response.content)['DescribeStackResourcesResponse']
resources = response_dict['DescribeStackResourcesResult']['StackResources']
if not resources:
# Check if stack exists
stack_name = req_data.get('StackName')[0]
cloudformation_client = aws_stack.connect_to_service('cloudformation')
try:
cloudformation_client.describe_stacks(StackName=stack_name)
except Exception:
return error_response('Stack with id %s does not exist' % stack_name, code=404)
if action == 'DescribeStackResource':
if response.status_code >= 500:
# fix an error in moto where it fails with 500 if the stack does not exist
return error_response('Stack resource does not exist', code=404)
if action == 'ListStackResources':
response_dict = xmltodict.parse(response.content, force_list=('member'))['ListStackResourcesResponse']
resources = response_dict['ListStackResourcesResult']['StackResourceSummaries']
if resources:
sqs_client = aws_stack.connect_to_service('sqs')
content_str = content_str_original = to_str(response.content)
new_response = Response()
new_response.status_code = response.status_code
new_response.headers = response.headers
for resource in resources['member']:
if resource['ResourceType'] == 'AWS::SQS::Queue':
try:
queue_name = resource['PhysicalResourceId']
queue_url = sqs_client.get_queue_url(QueueName=queue_name)['QueueUrl']
except Exception:
stack_name = req_data.get('StackName')[0]
return error_response('Stack with id %s does not exist' % stack_name, code=404)
content_str = re.sub(resource['PhysicalResourceId'], queue_url, content_str)
new_response._content = content_str
if content_str_original != new_response._content:
# if changes have been made, return patched response
new_response.headers['content-length'] = len(new_response._content)
return new_response
elif action in ('CreateStack', 'UpdateStack'):
if response.status_code >= 400:
return response
# run the actual deployment
template = template_deployer.template_to_json(req_data.get('TemplateBody')[0])
template_deployer.deploy_template(template, req_data.get('StackName')[0])
def forward_request(self, method, path, data, headers):
req_data = None
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action')[0]
if req_data:
if action == 'CreateStack':
return create_stack(req_data)
if action == 'CreateChangeSet':
return create_change_set(req_data)
elif action == 'DescribeChangeSet':
return describe_change_set(req_data)
elif action == 'ExecuteChangeSet':
return execute_change_set(req_data)
elif action == 'UpdateStack' and req_data.get('TemplateURL'):
# Temporary fix until the moto CF backend can handle TemplateURL (currently fails)
url = re.sub(r'https?://s3\.amazonaws\.com', aws_stack.get_local_service_url('s3'),
req_data.get('TemplateURL')[0])
req_data['TemplateBody'] = requests.get(url).content
modified_data = urlparse.urlencode(req_data, doseq=True)
return Request(data=modified_data, headers=headers, method=method)
elif action == 'ValidateTemplate':
return validate_template(req_data)
return True
def update_function_configuration(function):
""" Update the configuration of an existing function
---
operationId: 'updateFunctionConfiguration'
parameters:
- name: 'request'
in: body
"""
data = json.loads(to_str(request.data))
arn = func_arn(function)
# Stop/remove any containers that this arn uses.
LAMBDA_EXECUTOR.cleanup(arn)
lambda_details = arn_to_lambda[arn]
if data.get('Handler'):
lambda_details.handler = data['Handler']
if data.get('Runtime'):
lambda_details.runtime = data['Runtime']
if data.get('Environment'):
lambda_details.envvars = data.get('Environment', {}).get('Variables', {})
if data.get('Timeout'):
lambda_details.timeout = data['Timeout']
result = {}
return jsonify(result)
def invoke_function(function):
""" Invoke an existing function
---
operationId: 'invokeFunction'
parameters:
- name: 'request'
in: body
"""
arn = func_arn(function)
if arn not in arn_to_lambda:
return error_response('Function does not exist: %s' % arn, 404, error_type='ResourceNotFoundException')
qualifier = request.args['Qualifier'] if 'Qualifier' in request.args else '$LATEST'
if not arn_to_lambda.get(arn).qualifier_exists(qualifier):
return error_response('Function does not exist: {0}:{1}'.format(arn, qualifier), 404,
error_type='ResourceNotFoundException')
data = None
if request.data:
try:
data = json.loads(to_str(request.data))
except Exception:
return error_response('The payload is not JSON', 415, error_type='UnsupportedMediaTypeException')
async = False
if 'HTTP_X_AMZ_INVOCATION_TYPE' in request.environ:
async = request.environ['HTTP_X_AMZ_INVOCATION_TYPE'] == 'Event'
result = run_lambda(async=async, func_arn=arn, event=data, context={}, version=qualifier)
if isinstance(result, dict):
return jsonify(result)
if result:
return result
return make_response('', 200)
def get_metric_statistics(Namespace, MetricName, Dimensions,
Period=60, StartTime=None, EndTime=None, Statistics=None):
if not StartTime:
StartTime = datetime.now() - timedelta(minutes=5)
if not EndTime:
EndTime = datetime.now()
if Statistics is None:
Statistics = ['Sum']
cloudwatch_url = aws_stack.get_local_service_url('cloudwatch')
url = '%s/?Action=GetMetricValues' % cloudwatch_url
all_metrics = make_http_request(url)
assert all_metrics.status_code == 200
datapoints = []
for datapoint in json.loads(to_str(all_metrics.content)):
if datapoint['Namespace'] == Namespace and datapoint['Name'] == MetricName:
dp_dimensions = datapoint['Dimensions']
all_present = all(m in dp_dimensions for m in Dimensions)
no_additional = all(m in Dimensions for m in dp_dimensions)
if all_present and no_additional:
datapoints.append(datapoint)
result = {
'Label': '%s/%s' % (Namespace, MetricName),
'Datapoints': datapoints
}
return result
def test_firehose_s3():
s3_resource = aws_stack.connect_to_resource('s3')
firehose = aws_stack.connect_to_service('firehose')
s3_prefix = '/testdata'
test_data = '{"test": "firehose_data_%s"}' % short_uid()
# create Firehose stream
stream = firehose.create_delivery_stream(
DeliveryStreamName=TEST_FIREHOSE_NAME,
S3DestinationConfiguration={
'RoleARN': aws_stack.iam_resource_arn('firehose'),
'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME),
'Prefix': s3_prefix
}
)
assert stream
assert TEST_FIREHOSE_NAME in firehose.list_delivery_streams()['DeliveryStreamNames']
# create target S3 bucket
s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME)
# put records
firehose.put_record(
DeliveryStreamName=TEST_FIREHOSE_NAME,
Record={
'Data': to_bytes(test_data)
}
)
# check records in target bucket
all_objects = testutil.list_all_s3_objects()
testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
def get_machine_id():
global MACHINE_ID
if MACHINE_ID:
return MACHINE_ID
# determine MACHINE_ID from config files
configs_map = {}
config_file_tmp = get_config_file_tempdir()
config_file_home = get_config_file_homedir()
for config_file in (config_file_home, config_file_tmp):
if config_file:
local_configs = load_file(config_file)
local_configs = json.loads(to_str(local_configs))
configs_map[config_file] = local_configs
if 'machine_id' in local_configs:
MACHINE_ID = local_configs['machine_id']
break
# if we can neither find NOR create the config files, fall back to process id
if not configs_map:
return PROCESS_ID
# assign default id if empty
if not MACHINE_ID:
MACHINE_ID = short_uid()
# update MACHINE_ID in all config files
for config_file, configs in configs_map.items():
configs['machine_id'] = MACHINE_ID
save_file(config_file, json.dumps(configs))
return MACHINE_ID
def post_request():
action = request.headers.get('x-amz-target')
data = json.loads(to_str(request.data))
result = {}
kinesis = aws_stack.connect_to_service('kinesis')
if action == '%s.ListStreams' % ACTION_HEADER_PREFIX:
result = {
'Streams': list(DDB_STREAMS.values()),
'LastEvaluatedStreamArn': 'TODO'
}
elif action == '%s.DescribeStream' % ACTION_HEADER_PREFIX:
for stream in DDB_STREAMS.values():
if stream['StreamArn'] == data['StreamArn']:
result = {
'StreamDescription': stream
}
# get stream details
dynamodb = aws_stack.connect_to_service('dynamodb')
table_name = table_name_from_stream_arn(stream['StreamArn'])
stream_name = get_kinesis_stream_name(table_name)
stream_details = kinesis.describe_stream(StreamName=stream_name)
table_details = dynamodb.describe_table(TableName=table_name)
stream['KeySchema'] = table_details['Table']['KeySchema']
# Replace Kinesis ShardIDs with ones that mimic actual
# DynamoDBStream ShardIDs.
stream_shards = stream_details['StreamDescription']['Shards']
for shard in stream_shards:
shard['ShardId'] = shard_id(stream_name, shard['ShardId'])
stream['Shards'] = stream_shards
break
if not result:
return error_response('Requested resource not found', error_type='ResourceNotFoundException')
elif action == '%s.GetShardIterator' % ACTION_HEADER_PREFIX:
# forward request to Kinesis API
stream_name = stream_name_from_stream_arn(data['StreamArn'])
stream_shard_id = kinesis_shard_id(data['ShardId'])
result = kinesis.get_shard_iterator(StreamName=stream_name,
ShardId=stream_shard_id, ShardIteratorType=data['ShardIteratorType'])
elif action == '%s.GetRecords' % ACTION_HEADER_PREFIX:
kinesis_records = kinesis.get_records(**data)
result = {'Records': []}
for record in kinesis_records['Records']:
result['Records'].append(json.loads(to_str(record['Data'])))
else:
print('WARNING: Unknown operation "%s"' % action)
return jsonify(result)
def set_lifecycle(bucket_name, lifecycle):
# TODO: check if bucket exists, otherwise return 404-like error
if isinstance(to_str(lifecycle), six.string_types):
lifecycle = xmltodict.parse(lifecycle)
BUCKET_LIFECYCLE[bucket_name] = lifecycle
response = Response()
response.status_code = 200
return response
def create_domain():
data = json.loads(to_str(request.data))
domain_name = data['DomainName']
if domain_name in ES_DOMAINS:
return error_response(error_type='ResourceAlreadyExistsException')
ES_DOMAINS[domain_name] = data
result = get_domain_status(domain_name)
return jsonify(result)
def post_request():
action = request.headers.get('x-amz-target')
data = json.loads(to_str(request.data))
response = None
if action == '%s.ListDeliveryStreams' % ACTION_HEADER_PREFIX:
response = {
'DeliveryStreamNames': get_delivery_stream_names(),
'HasMoreDeliveryStreams': False
}
elif action == '%s.CreateDeliveryStream' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
response = create_stream(stream_name,
s3_destination=data.get('S3DestinationConfiguration'),
elasticsearch_destination=data.get('ElasticsearchDestinationConfiguration'))
elif action == '%s.DeleteDeliveryStream' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
response = delete_stream(stream_name)
elif action == '%s.DescribeDeliveryStream' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
response = get_stream(stream_name)
if not response:
return error_not_found(stream_name)
response = {
'DeliveryStreamDescription': response
}
elif action == '%s.PutRecord' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
record = data['Record']
put_record(stream_name, record)
response = {
'RecordId': str(uuid.uuid4())
}
elif action == '%s.PutRecordBatch' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
records = data['Records']
put_records(stream_name, records)
response = {
'FailedPutCount': 0,
'RequestResponses': []
}
elif action == '%s.UpdateDestination' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
version_id = data['CurrentDeliveryStreamVersionId']
destination_id = data['DestinationId']
s3_update = data['S3DestinationUpdate'] if 'S3DestinationUpdate' in data else None
update_destination(stream_name=stream_name, destination_id=destination_id,
s3_update=s3_update, version_id=version_id)
es_update = data['ESDestinationUpdate'] if 'ESDestinationUpdate' in data else None
update_destination(stream_name=stream_name, destination_id=destination_id,
es_update=es_update, version_id=version_id)
response = {}
else:
response = error_response('Unknown action "%s"' % action, code=400, error_type='InvalidAction')
if isinstance(response, dict):
response = jsonify(response)
return response
def receive_assert_delete(queue_url, assertions, sqs_client=None, required_subject=None):
if not sqs_client:
sqs_client = aws_stack.connect_to_service('sqs')
response = sqs_client.receive_message(QueueUrl=queue_url)
messages = [json.loads(to_str(m['Body'])) for m in response['Messages']]
testutil.assert_objects(assertions, messages)
for message in response['Messages']:
sqs_client.delete_message(QueueUrl=queue_url, ReceiptHandle=message['ReceiptHandle'])
def test_message_transformation():
template = APIGATEWAY_TRANSFORMATION_TEMPLATE
records = [
{
'data': {
'foo': 'foo1',
'bar': 'bar2'
}
},
{
'data': {
'foo': 'foo1',
'bar': 'bar2'
},
'partitionKey': 'key123'
}
]
context = {
'records': records
}
# try rendering the template
result = render_velocity_template(template, context, as_json=True)
result_decoded = json.loads(to_str(base64.b64decode(result['Records'][0]['Data'])))
assert result_decoded == records[0]['data']
assert len(result['Records'][0]['PartitionKey']) > 0
assert result['Records'][1]['PartitionKey'] == 'key123'
# try again with context as string
context = json.dumps(context)
result = render_velocity_template(template, context, as_json=True)
result_decoded = json.loads(to_str(base64.b64decode(result['Records'][0]['Data'])))
assert result_decoded == records[0]['data']
assert len(result['Records'][0]['PartitionKey']) > 0
assert result['Records'][1]['PartitionKey'] == 'key123'
# test with empty array
records = []
context = {
'records': records
}
# try rendering the template
result = render_velocity_template(template, context, as_json=True)
assert result['Records'] == []
def update_function_code(function):
""" Update the code of an existing function
---
operationId: 'updateFunctionCode'
parameters:
- name: 'request'
in: body
"""
data = json.loads(to_str(request.data))
result = set_function_code(data, function)
return jsonify(result or {})
def forward_request(self, method, path, data, headers):
data = json.loads(to_str(data))
if random.random() < config.DYNAMODB_ERROR_PROBABILITY:
return error_response_throughput()
action = headers.get('X-Amz-Target')
if action in ('%s.PutItem' % ACTION_PREFIX, '%s.UpdateItem' % ACTION_PREFIX, '%s.DeleteItem' % ACTION_PREFIX):
# find an existing item and store it in a thread-local, so we can access it in return_response,
# in order to determine whether an item already existed (MODIFY) or not (INSERT)
ProxyListenerDynamoDB.thread_local.existing_item = find_existing_item(data)
elif action == '%s.UpdateTimeToLive' % ACTION_PREFIX:
# TODO: TTL status is maintained/mocked but no real expiry is happening for items
response = Response()
response.status_code = 200
self._table_ttl_map[data['TableName']] = {
'AttributeName': data['TimeToLiveSpecification']['AttributeName'],
'Status': data['TimeToLiveSpecification']['Enabled']
}
response._content = json.dumps({'TimeToLiveSpecification': data['TimeToLiveSpecification']})
fix_headers_for_updated_response(response)
return response
elif action == '%s.DescribeTimeToLive' % ACTION_PREFIX:
response = Response()
response.status_code = 200
if data['TableName'] in self._table_ttl_map:
if self._table_ttl_map[data['TableName']]['Status']:
ttl_status = 'ENABLED'
else:
ttl_status = 'DISABLED'
response._content = json.dumps({
'TimeToLiveDescription': {
'AttributeName': self._table_ttl_map[data['TableName']]['AttributeName'],
'TimeToLiveStatus': ttl_status
}
})
else: # TTL for dynamodb table not set
response._content = json.dumps({'TimeToLiveDescription': {'TimeToLiveStatus': 'DISABLED'}})
fix_headers_for_updated_response(response)
return response
elif action == '%s.TagResource' % ACTION_PREFIX or action == '%s.UntagResource' % ACTION_PREFIX:
response = Response()
response.status_code = 200
response._content = '' # returns an empty body on success.
fix_headers_for_updated_response(response)
return response
elif action == '%s.ListTagsOfResource' % ACTION_PREFIX:
response = Response()
response.status_code = 200
response._content = json.dumps({'Tags': []}) # TODO: mocked and returns an empty list of tags for now.
fix_headers_for_updated_response(response)
return response
return True
def create_event_source_mapping():
""" Create new event source mapping
---
operationId: 'createEventSourceMapping'
parameters:
- name: 'request'
in: body
"""
data = json.loads(to_str(request.data))
mapping = add_event_source(data['FunctionName'], data['EventSourceArn'])
return jsonify(mapping)
def return_response(self, method, path, data, headers, response):
# This method is executed by the proxy after we've already received a
# response from the backend, hence we can utilize the "response" variable here
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
req_action = req_data['Action'][0]
if req_action == 'Subscribe' and response.status_code < 400:
response_data = xmltodict.parse(response.content)
topic_arn = (req_data.get('TargetArn') or req_data.get('TopicArn'))[0]
sub_arn = response_data['SubscribeResponse']['SubscribeResult']['SubscriptionArn']
do_subscribe(topic_arn, req_data['Endpoint'][0], req_data['Protocol'][0], sub_arn)
def test_api_gateway_http_integration():
test_port = 12123
backend_url = 'http://localhost:%s%s' % (test_port, API_PATH_HTTP_BACKEND)
# create target HTTP backend
class TestListener(ProxyListener):
def forward_request(self, **kwargs):
response = Response()
response.status_code = 200
response._content = kwargs.get('data') or '{}'
return response
proxy = GenericProxy(test_port, update_listener=TestListener())
proxy.start()
# create API Gateway and connect it to the HTTP backend
result = connect_api_gateway_to_http('test_gateway2', backend_url, path=API_PATH_HTTP_BACKEND)
url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'],
stage_name=TEST_STAGE_NAME, path=API_PATH_HTTP_BACKEND)
# make sure CORS headers are present
origin = 'localhost'
result = requests.options(url, headers={'origin': origin})
assert result.status_code == 200
assert re.match(result.headers['Access-Control-Allow-Origin'].replace('*', '.*'), origin)
assert 'POST' in result.headers['Access-Control-Allow-Methods']
# make test request to gateway
result = requests.get(url)
assert result.status_code == 200
assert to_str(result.content) == '{}'
data = {'data': 123}
result = requests.post(url, data=json.dumps(data))
assert result.status_code == 200
assert json.loads(to_str(result.content)) == data
# clean up
proxy.stop()
def forward_request(self, method, path, data, headers):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
if 'QueueName' in req_data:
if '.' in req_data['QueueName'][0]:
# ElasticMQ currently does not support "." in the queue name, e.g., for *.fifo queues
# TODO: remove this once *.fifo queues are supported in ElasticMQ
req_data['QueueName'][0] = req_data['QueueName'][0].replace('.', '_')
modified_data = urlencode(req_data, doseq=True)
request = Request(data=modified_data, headers=headers, method=method)
return request
return True
def deserialize_event(event):
# Deserialize into Python dictionary and extract the "NewImage" (the new version of the full ddb document)
ddb = event.get('dynamodb')
if ddb:
ddb_deserializer = TypeDeserializer()
result = ddb_deserializer.deserialize({'M': ddb.get('NewImage')})
result['__action_type'] = event.get('eventName')
return result
kinesis = event.get('kinesis')
if kinesis:
assert kinesis['sequenceNumber']
kinesis['data'] = json.loads(to_str(base64.b64decode(kinesis['data'])))
return kinesis
return event.get('Sns')
def create_function():
""" Create new function
---
operationId: 'createFunction'
parameters:
- name: 'request'
in: body
"""
arn = 'n/a'
try:
data = json.loads(to_str(request.data))
lambda_name = data['FunctionName']
event_publisher.fire_event(event_publisher.EVENT_LAMBDA_CREATE_FUNC,
payload={'n': event_publisher.get_hash(lambda_name)})
arn = func_arn(lambda_name)
if arn in arn_to_lambda:
return error_response('Function already exist: %s' %
lambda_name, 409, error_type='ResourceConflictException')
arn_to_lambda[arn] = func_details = LambdaFunction(arn)
func_details.versions = {'$LATEST': {'CodeSize': 50}}
func_details.handler = data['Handler']
func_details.runtime = data['Runtime']
func_details.envvars = data.get('Environment', {}).get('Variables', {})
func_details.timeout = data.get('Timeout')
result = set_function_code(data['Code'], lambda_name)
if isinstance(result, Response):
del arn_to_lambda[arn]
return result
result.update({
'DeadLetterConfig': data.get('DeadLetterConfig'),
'Description': data.get('Description'),
'Environment': {'Error': {}, 'Variables': func_details.envvars},
'FunctionArn': arn,
'FunctionName': lambda_name,
'Handler': func_details.handler,
'MemorySize': data.get('MemorySize'),
'Role': data.get('Role'),
'Runtime': func_details.runtime,
'Timeout': data.get('Timeout'),
'TracingConfig': {},
'VpcConfig': {'SecurityGroupIds': [None], 'SubnetIds': [None], 'VpcId': None}
})
return jsonify(result or {})
except Exception as e:
del arn_to_lambda[arn]
return error_response('Unknown error: %s %s' % (e, traceback.format_exc()))
def kinesis_get_latest_records(stream_name, shard_id, count=10, env=None):
kinesis = connect_to_service('kinesis', env=env)
result = []
response = kinesis.get_shard_iterator(StreamName=stream_name, ShardId=shard_id,
ShardIteratorType='TRIM_HORIZON')
shard_iterator = response['ShardIterator']
while shard_iterator:
records_response = kinesis.get_records(ShardIterator=shard_iterator)
records = records_response['Records']
for record in records:
try:
record['Data'] = to_str(record['Data'])
except Exception:
pass
result.extend(records)
shard_iterator = records_response['NextShardIterator'] if records else False
while len(result) > count:
result.pop(0)
return result
def test_api_gateway_kinesis_integration():
# create target Kinesis stream
aws_stack.create_kinesis_stream(TEST_STREAM_KINESIS_API_GW)
# create API Gateway and connect it to the target stream
result = connect_api_gateway_to_kinesis('test_gateway1', TEST_STREAM_KINESIS_API_GW)
# generate test data
test_data = {'records': [
{'data': '{"foo": "bar1"}'},
{'data': '{"foo": "bar2"}'},
{'data': '{"foo": "bar3"}'}
]}
url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'],
stage_name=TEST_STAGE_NAME, path=API_PATH_DATA_INBOUND)
result = requests.post(url, data=json.dumps(test_data))
result = json.loads(to_str(result.content))
assert result['FailedRecordCount'] == 0
assert len(result['Records']) == len(test_data['records'])
def test_api_gateway_lambda_proxy_integration():
# create lambda function
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_PROXY_BACKEND,
zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(TEST_LAMBDA_PROXY_BACKEND)
target_uri = 'arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations' % (DEFAULT_REGION, lambda_uri)
result = connect_api_gateway_to_http_with_lambda_proxy('test_gateway2', target_uri,
path=API_PATH_LAMBDA_PROXY_BACKEND)
# make test request to gateway and check response
path = API_PATH_LAMBDA_PROXY_BACKEND.replace('{test_param1}', 'foo1')
url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'], stage_name=TEST_STAGE_NAME, path=path)
data = {'return_status_code': 203, 'return_headers': {'foo': 'bar123'}}
result = requests.post(url, data=json.dumps(data))
assert result.status_code == 203
assert result.headers.get('foo') == 'bar123'
parsed_body = json.loads(to_str(result.content))
assert parsed_body.get('return_status_code') == 203
assert parsed_body.get('return_headers') == {'foo': 'bar123'}
assert parsed_body.get('pathParameters') == {'test_param1': 'foo1'}
def get_payload(request):
return json.loads(common.to_str(request.data))
def _fix_error_codes(method, data, response):
if method == 'POST' and 'Action=CreateRole' in to_str(data) and response.status_code >= 400:
content = to_str(response.content)
flags = re.MULTILINE | re.DOTALL
# remove the <Errors> wrapper element, as this breaks AWS Java SDKs (issue #2231)
response._content = re.sub(r'<Errors>\s*(<Error>(\s|.)*</Error>)\s*</Errors>', r'\1', content, flags)
def return_response(self, method, path, data, headers, response, request_handler):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action', [None])[0]
event_type = None
queue_url = None
if action == 'CreateQueue':
event_type = event_publisher.EVENT_SQS_CREATE_QUEUE
response_data = xmltodict.parse(response.content)
if 'CreateQueueResponse' in response_data:
queue_url = response_data['CreateQueueResponse']['CreateQueueResult']['QueueUrl']
elif action == 'DeleteQueue':
event_type = event_publisher.EVENT_SQS_DELETE_QUEUE
queue_url = req_data.get('QueueUrl', [None])[0]
if event_type and queue_url:
event_publisher.fire_event(event_type, payload={'u': event_publisher.get_hash(queue_url)})
# patch the response and return the correct endpoint URLs
if action in ('CreateQueue', 'GetQueueUrl', 'ListQueues'):
content_str = content_str_original = to_str(response.content)
new_response = Response()
new_response.status_code = response.status_code
new_response.headers = response.headers
if config.USE_SSL and '<QueueUrl>http://' in content_str:
# return https://... if we're supposed to use SSL
content_str = re.sub(r'<QueueUrl>\s*http://', r'<QueueUrl>https://', content_str)
# expose external hostname:port
external_port = get_external_port(headers, request_handler)
content_str = re.sub(r'<QueueUrl>\s*([a-z]+)://[^<]*:([0-9]+)/([^<]*)\s*</QueueUrl>',
r'<QueueUrl>\1://%s:%s/\3</QueueUrl>' % (HOSTNAME_EXTERNAL, external_port), content_str)
new_response._content = content_str
if content_str_original != new_response._content:
# if changes have been made, return patched response
new_response.headers['content-length'] = len(new_response._content)
return new_response
# Since the following 2 API calls are not implemented in ElasticMQ, we're mocking them
# and letting them to return an empty response
if action == 'TagQueue':
new_response = Response()
new_response.status_code = 200
new_response._content = (
'<?xml version="1.0"?>'
'<TagQueueResponse>'
'<ResponseMetadata>' # noqa: W291
'<RequestId>{}</RequestId>' # noqa: W291
'</ResponseMetadata>' # noqa: W291
'</TagQueueResponse>'
).format(uuid.uuid4())
return new_response
elif action == 'ListQueueTags':
new_response = Response()
new_response.status_code = 200
new_response._content = (
'<?xml version="1.0"?>'
'<ListQueueTagsResponse xmlns="{}">'
'<ListQueueTagsResult/>' # noqa: W291
'<ResponseMetadata>' # noqa: W291
'<RequestId>{}</RequestId>' # noqa: W291
'</ResponseMetadata>' # noqa: W291
'</ListQueueTagsResponse>'
).format(XMLNS_SQS, uuid.uuid4())
return new_response
def forward_request(self, method, path, data, headers):
# check region
try:
aws_stack.check_valid_region(headers)
except Exception as e:
return make_error(message=str(e), code=400)
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
req_action = req_data['Action'][0]
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
do_create_topic(topic_arn)
if req_action == 'SetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
attr_name = req_data['AttributeName'][0]
attr_value = req_data['AttributeValue'][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == 'GetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
content = '<Attributes>'
for key, value in sub.items():
content += '<entry><key>%s</key><value>%s</value></entry>\n' % (
key, value)
content += '</Attributes>'
return make_response(req_action, content=content)
elif req_action == 'Subscribe':
if 'Endpoint' not in req_data:
return make_error(
message='Endpoint not specified in subscription',
code=400)
elif req_action == 'Unsubscribe':
if 'SubscriptionArn' not in req_data:
return make_error(
message=
'SubscriptionArn not specified in unsubscribe request',
code=400)
do_unsubscribe(req_data.get('SubscriptionArn')[0])
elif req_action == 'DeleteTopic':
do_delete_topic(topic_arn)
elif req_action == 'Publish':
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS.get(topic_arn, []):
filter_policy = json.loads(
subscriber.get('FilterPolicy', '{}'))
message_attributes = get_message_attributes(req_data)
if check_filter_policy(filter_policy, message_attributes):
if subscriber['Protocol'] == 'sqs':
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(
queue_name)
subscriber['sqs_queue_url'] = queue_url
try:
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(
subscriber, req_data),
MessageAttributes=
create_sqs_message_attributes(
subscriber, message_attributes))
except Exception as exc:
return make_error(message=str(exc), code=400)
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
message,
subject=req_data.get('Subject', [None])[0])
elif subscriber['Protocol'] in ['http', 'https']:
try:
message_body = create_sns_message_body(
subscriber, req_data)
except Exception as exc:
return make_error(message=str(exc), code=400)
requests.post(subscriber['Endpoint'],
headers={
'Content-Type':
'text/plain',
'x-amz-sns-message-type':
'Notification'
},
data=message_body)
else:
LOGGER.warning(
'Unexpected protocol "%s" for SNS subscription'
% subscriber['Protocol'])
# return response here because we do not want the request to be forwarded to SNS
return make_response(req_action)
return True
def return_response(self, method, path, data, headers, response,
request_handler):
if method == 'OPTIONS' and path == '/':
# Allow CORS preflight requests to succeed.
return 200
if method != 'POST':
return
region_name = extract_region_from_auth_header(headers)
req_data = urlparse.parse_qs(to_str(data))
action = req_data.get('Action', [None])[0]
content_str = content_str_original = to_str(response.content)
self._fire_event(req_data, response)
# patch the response and add missing attributes
if action == 'GetQueueAttributes':
content_str = self._add_queue_attributes(path, req_data,
content_str, headers)
# patch the response and return the correct endpoint URLs / ARNs
if action in ('CreateQueue', 'GetQueueUrl', 'ListQueues',
'GetQueueAttributes'):
if config.USE_SSL and '<QueueUrl>http://' in content_str:
# return https://... if we're supposed to use SSL
content_str = re.sub(r'<QueueUrl>\s*http://',
r'<QueueUrl>https://', content_str)
# expose external hostname:port
external_port = SQS_PORT_EXTERNAL or get_external_port(
headers, request_handler)
content_str = re.sub(
r'<QueueUrl>\s*([a-z]+)://[^<]*:([0-9]+)/([^<]*)\s*</QueueUrl>',
r'<QueueUrl>\1://%s:%s/\3</QueueUrl>' %
(HOSTNAME_EXTERNAL, external_port), content_str)
# fix queue ARN
content_str = re.sub(
r'<([a-zA-Z0-9]+)>\s*arn:aws:sqs:elasticmq:([^<]+)</([a-zA-Z0-9]+)>',
r'<\1>arn:aws:sqs:%s:\2</\3>' % (region_name), content_str)
if content_str_original != content_str:
# if changes have been made, return patched response
new_response = Response()
new_response.status_code = response.status_code
new_response.headers = response.headers
new_response._content = content_str
new_response.headers['content-length'] = len(new_response._content)
return new_response
# Since the following 2 API calls are not implemented in ElasticMQ, we're mocking them
# and letting them to return an empty response
if action == 'TagQueue':
new_response = Response()
new_response.status_code = 200
new_response._content = ("""
<?xml version="1.0"?>
<TagQueueResponse>
<ResponseMetadata>
<RequestId>{}</RequestId>
</ResponseMetadata>
</TagQueueResponse>
""").strip().format(uuid.uuid4())
return new_response
elif action == 'ListQueueTags':
new_response = Response()
new_response.status_code = 200
new_response._content = ("""
<?xml version="1.0"?>
<ListQueueTagsResponse xmlns="{}">
<ListQueueTagsResult/>
<ResponseMetadata>
<RequestId>{}</RequestId>
</ResponseMetadata>
</ListQueueTagsResponse>
""").strip().format(XMLNS_SQS, uuid.uuid4())
return new_response
def return_response(self, method, path, data, headers, response):
data = json.loads(to_str(data))
# update table definitions
if data and 'TableName' in data and 'KeySchema' in data:
TABLE_DEFINITIONS[data['TableName']] = data
if response._content:
# fix the table ARN (DynamoDBLocal hardcodes "ddblocal" as the region)
content_replaced = re.sub(
r'"TableArn"\s*:\s*"arn:aws:dynamodb:ddblocal:([^"]+)"',
r'"TableArn": "arn:aws:dynamodb:%s:\1"' %
aws_stack.get_local_region(), to_str(response._content))
if content_replaced != response._content:
response._content = content_replaced
fix_headers_for_updated_response(response)
action = headers.get('X-Amz-Target')
if not action:
return
record = {
'eventID': '1',
'eventVersion': '1.0',
'dynamodb': {
'StreamViewType': 'NEW_AND_OLD_IMAGES',
'SizeBytes': -1
},
'awsRegion': DEFAULT_REGION,
'eventSource': 'aws:dynamodb'
}
records = [record]
if action == '%s.UpdateItem' % ACTION_PREFIX:
updated_item = find_existing_item(data)
if not updated_item:
return
record['eventName'] = 'MODIFY'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb'][
'OldImage'] = ProxyListenerDynamoDB.thread_local.existing_item
record['dynamodb']['NewImage'] = updated_item
record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
records = []
for table_name, requests in data['RequestItems'].items():
for request in requests:
put_request = request.get('PutRequest')
if put_request:
keys = dynamodb_extract_keys(item=put_request['Item'],
table_name=table_name)
if isinstance(keys, Response):
return keys
new_record = clone(record)
new_record['eventName'] = 'INSERT'
new_record['dynamodb']['Keys'] = keys
new_record['dynamodb']['NewImage'] = put_request[
'Item']
new_record[
'eventSourceARN'] = aws_stack.dynamodb_table_arn(
table_name)
records.append(new_record)
elif action == '%s.PutItem' % ACTION_PREFIX:
existing_item = ProxyListenerDynamoDB.thread_local.existing_item
ProxyListenerDynamoDB.thread_local.existing_item = None
record['eventName'] = 'INSERT' if not existing_item else 'MODIFY'
keys = dynamodb_extract_keys(item=data['Item'],
table_name=data['TableName'])
if isinstance(keys, Response):
return keys
record['dynamodb']['Keys'] = keys
record['dynamodb']['NewImage'] = data['Item']
record['dynamodb']['SizeBytes'] = len(json.dumps(data['Item']))
elif action == '%s.GetItem' % ACTION_PREFIX:
if response.status_code == 200:
content = json.loads(to_str(response.content))
# make sure we append 'ConsumedCapacity', which is properly
# returned by dynalite, but not by AWS's DynamoDBLocal
if 'ConsumedCapacity' not in content and data.get(
'ReturnConsumedCapacity') in ('TOTAL', 'INDEXES'):
content['ConsumedCapacity'] = {
'CapacityUnits': 0.5, # TODO hardcoded
'TableName': data['TableName']
}
response._content = json.dumps(content)
fix_headers_for_updated_response(response)
elif action == '%s.DeleteItem' % ACTION_PREFIX:
record['eventName'] = 'REMOVE'
record['dynamodb']['Keys'] = data['Key']
elif action == '%s.CreateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_CREATE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.DeleteTable' % ACTION_PREFIX:
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_DELETE_TABLE,
payload={'n': event_publisher.get_hash(data['TableName'])})
return
elif action == '%s.UpdateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
create_dynamodb_stream(data)
return
else:
# nothing to do
return
if len(records) > 0 and 'eventName' in records[0]:
if 'TableName' in data:
records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn(
data['TableName'])
forward_to_lambda(records)
forward_to_ddb_stream(records)
def decode_content(self, data):
try:
return json.loads(to_str(data))
except UnicodeDecodeError:
return cbor2.loads(data)
def _escape(val):
try:
return val and escape(to_str(val))
except Exception:
return val
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(to_str(data))
print('ProxyListenerKinesis return_response data: [', data,
'] action [', action, ']')
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM
if action == ACTION_CREATE_STREAM else
event_publisher.EVENT_KINESIS_DELETE_STREAM)
payload = {'n': event_publisher.get_hash(data.get('StreamName'))}
if action == ACTION_CREATE_STREAM:
payload['s'] = data.get('ShardCount')
print(
'ProxyListenerKinesis return_response ACTION_CREATE_STREAM, ACTION_DELETE_STREAM '
+ 'event_publisher.fire_event event_type: [', event_type,
'] payload[', payload, ']')
event_publisher.fire_event(event_type, payload=payload)
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
print(
'ProxyListenerKinesis return_response ACTION_PUT_RECORD: ' +
'lambda_api.process_kinesis_records event_record[',
event_record, 'stream_name[', stream_name, ']')
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
if 'Records' in response_body:
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data':
record['Data'],
'partitionKey':
record['PartitionKey'],
'sequenceNumber':
response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
print(
'ProxyListenerKinesis return_response ACTION_PUT_RECORDS: '
+ 'lambda_api.process_kinesis_records event_record[',
event_records, 'stream_name[', stream_name, ']')
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_UPDATE_SHARD_COUNT:
# Currently kinesalite, which backs the Kinesis implementation for localstack, does
# not support UpdateShardCount:
# https://github.com/mhart/kinesalite/issues/61
#
# [Terraform](https://www.terraform.io) makes the call to UpdateShardCount when it
# applies Kinesis resources. A Terraform run fails when this is not present.
#
# The code that follows just returns a successful response, bypassing the 400
# response that kinesalite returns.
#
response = Response()
response.status_code = 200
content = {
'CurrentShardCount': 1,
'StreamName': data['StreamName'],
'TargetShardCount': data['TargetShardCount']
}
response.encoding = 'UTF-8'
response._content = json.dumps(content)
print(
'ProxyListenerKinesis return_response ACTION_UPDATE_SHARD_COUNT: response[',
response, ']')
return response
def return_response(self, method, path, data, headers, response,
request_handler):
# persist requests to disk
super(ProxyListenerSQS,
self).return_response(method, path, data, headers, response,
request_handler)
if method == 'OPTIONS' and path == '/':
# Allow CORS preflight requests to succeed.
return 200
if method != 'POST':
return
region_name = aws_stack.get_region()
req_data = parse_request_data(method, path, data)
action = req_data.get('Action')
content_str = content_str_original = to_str(response.content)
if response.status_code >= 400:
return response
_fire_event(req_data, response)
# patch the response and add missing attributes
if action == 'GetQueueAttributes':
content_str = _add_queue_attributes(path, req_data, content_str,
headers)
# patch the response and return the correct endpoint URLs / ARNs
if action in ('CreateQueue', 'GetQueueUrl', 'ListQueues',
'GetQueueAttributes'):
if config.USE_SSL and '<QueueUrl>http://' in content_str:
# return https://... if we're supposed to use SSL
content_str = re.sub(r'<QueueUrl>\s*http://',
r'<QueueUrl>https://', content_str)
# expose external hostname:port
external_port = SQS_PORT_EXTERNAL or get_external_port(
headers, request_handler)
content_str = re.sub(
r'<QueueUrl>\s*([a-z]+)://[^<]*:([0-9]+)/([^<]*)\s*</QueueUrl>',
r'<QueueUrl>\1://%s:%s/\3</QueueUrl>' %
(HOSTNAME_EXTERNAL, external_port), content_str)
# encode account ID in queue URL
content_str = re.sub(
r'<QueueUrl>\s*([a-z]+)://([^/]+)/queue/([^<]*)\s*</QueueUrl>',
r'<QueueUrl>\1://\2/%s/\3</QueueUrl>' %
constants.TEST_AWS_ACCOUNT_ID, content_str)
# fix queue ARN
content_str = re.sub(
r'<([a-zA-Z0-9]+)>\s*arn:aws:sqs:elasticmq:([^<]+)</([a-zA-Z0-9]+)>',
r'<\1>arn:aws:sqs:%s:\2</\3>' % region_name, content_str)
if action == 'CreateQueue':
queue_url = re.match(r'.*<QueueUrl>(.*)</QueueUrl>',
content_str, re.DOTALL).group(1)
_set_queue_attributes(queue_url, req_data)
elif action == 'SendMessageBatch':
if validate_empty_message_batch(data, req_data):
msg = 'There should be at least one SendMessageBatchRequestEntry in the request.'
return make_requests_error(code=404,
code_string='EmptyBatchRequest',
message=msg)
# instruct listeners to fetch new SQS message
if action in ('SendMessage', 'SendMessageBatch'):
_process_sent_message(path, req_data, headers)
if content_str_original != content_str:
# if changes have been made, return patched response
response.headers['content-length'] = len(content_str)
return requests_response(content_str,
headers=response.headers,
status_code=response.status_code)
def validate_empty_message_batch(data, req_data):
data = to_str(data).split('Entries=')
if len(data) > 1 and not req_data.get('Entries'):
return True
return False
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
return 200
def test_apigateway_with_lambda_integration(self):
apigw_client = aws_stack.connect_to_service('apigateway')
# create Lambda function
lambda_name = 'apigw-lambda-%s' % short_uid()
self.create_lambda_function(lambda_name)
lambda_uri = aws_stack.lambda_function_arn(lambda_name)
target_uri = aws_stack.apigateway_invocations_arn(lambda_uri)
# create REST API
api = apigw_client.create_rest_api(name='test-api', description='')
api_id = api['id']
root_res_id = apigw_client.get_resources(restApiId=api_id)['items'][0]['id']
api_resource = apigw_client.create_resource(restApiId=api_id, parentId=root_res_id, pathPart='test')
apigw_client.put_method(
restApiId=api_id,
resourceId=api_resource['id'],
httpMethod='GET',
authorizationType='NONE'
)
rs = apigw_client.put_integration(
restApiId=api_id,
resourceId=api_resource['id'],
httpMethod='GET',
integrationHttpMethod='POST',
type='AWS',
uri=target_uri,
timeoutInMillis=3000,
contentHandling='CONVERT_TO_BINARY',
requestTemplates={
'application/json': '{"param1": "$input.params(\'param1\')"}'
}
)
integration_keys = ['httpMethod', 'type', 'passthroughBehavior', 'cacheKeyParameters', 'uri', 'cacheNamespace',
'timeoutInMillis', 'contentHandling', 'requestParameters']
self.assertEqual(rs['ResponseMetadata']['HTTPStatusCode'], 200)
for key in integration_keys:
self.assertIn(key, rs)
self.assertNotIn('responseTemplates', rs)
apigw_client.create_deployment(restApiId=api_id, stageName=self.TEST_STAGE_NAME)
rs = apigw_client.get_integration(
restApiId=api_id,
resourceId=api_resource['id'],
httpMethod='GET'
)
self.assertEqual(rs['ResponseMetadata']['HTTPStatusCode'], 200)
self.assertEqual(rs['type'], 'AWS')
self.assertEqual(rs['httpMethod'], 'POST')
self.assertEqual(rs['uri'], target_uri)
# invoke the gateway endpoint
url = gateway_request_url(api_id=api_id, stage_name=self.TEST_STAGE_NAME, path='/test')
response = requests.get('%s?param1=foobar' % url)
self.assertLess(response.status_code, 400)
content = json.loads(to_str(response.content))
self.assertEqual(content.get('httpMethod'), 'GET')
self.assertEqual(content.get('requestContext', {}).get('resourceId'), api_resource['id'])
self.assertEqual(content.get('requestContext', {}).get('stage'), self.TEST_STAGE_NAME)
self.assertEqual(content.get('body'), '{"param1": "foobar"}')
# delete integration
rs = apigw_client.delete_integration(
restApiId=api_id,
resourceId=api_resource['id'],
httpMethod='GET',
)
self.assertEqual(rs['ResponseMetadata']['HTTPStatusCode'], 200)
with self.assertRaises(ClientError) as ctx:
# This call should not be successful as the integration is deleted
apigw_client.get_integration(
restApiId=api_id,
resourceId=api_resource['id'],
httpMethod='GET'
)
self.assertEqual(ctx.exception.response['Error']['Code'], 'BadRequestException')
# clean up
lambda_client = aws_stack.connect_to_service('lambda')
lambda_client.delete_function(FunctionName=lambda_name)
apigw_client.delete_rest_api(restApiId=api_id)
def test_lambda_runtimes():
lambda_client = aws_stack.connect_to_service('lambda')
# deploy and invoke lambda - Python 2.7
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON27)
# Invocation Type not set
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY,
Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# Invocation Type - RequestResponse
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY,
Payload=b'{}',
InvocationType='RequestResponse')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# Invocation Type - Event
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY,
Payload=b'{}',
InvocationType='Event')
assert result['StatusCode'] == 202
# Invocation Type - DryRun
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY,
Payload=b'{}',
InvocationType='DryRun')
assert result['StatusCode'] == 204
if use_docker():
# deploy and invoke lambda - Python 3.6
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON3),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON36)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON36)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3,
Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke lambda - Java
if not os.path.exists(TEST_LAMBDA_JAVA):
mkdir(os.path.dirname(TEST_LAMBDA_JAVA))
download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA)
# Lambda supports single JAR deployments without the zip, so we upload the JAR directly.
test_java_jar = load_file(TEST_LAMBDA_JAVA, mode='rb')
assert test_java_jar is not None
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JAVA,
zip_file=test_java_jar,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.LambdaHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA,
Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert 'LinkedHashMap' in to_str(result_data)
# test SNSEvent
result = lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_JAVA,
InvocationType='Event',
Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}')
assert result['StatusCode'] == 202
# test DDBEvent
result = lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_JAVA,
InvocationType='Event',
Payload=b'{"Records": [{"dynamodb": {"Message": "{}"}}]}')
assert result['StatusCode'] == 202
# test KinesisEvent
result = lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_JAVA,
Payload=
b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}'
)
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert 'KinesisEvent' in to_str(result_data)
# deploy and invoke lambda - Java with stream handler
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JAVA_STREAM,
zip_file=test_java_jar,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.LambdaStreamHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM,
Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke lambda - Java with serializable input object
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE,
zip_file=test_java_jar,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.SerializedInputLambdaHandler')
result = lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE,
Payload=b'{"bucket": "test_bucket", "key": "test_key"}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {
'validated': True,
'bucket': 'test_bucket',
'key': 'test_key'
}
if use_docker():
# deploy and invoke lambda - Node.js
zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS,
get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS,
zip_file=zip_file,
handler='lambda_integration.handler',
runtime=LAMBDA_RUNTIME_NODEJS)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS,
Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke - .NET Core 2.0. Its already a zip
zip_file = TEST_LAMBDA_DOTNETCORE2
zip_file_content = None
with open(zip_file, 'rb') as file_obj:
zip_file_content = file_obj.read()
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_DOTNETCORE2,
zip_file=zip_file_content,
handler=
'DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler',
runtime=LAMBDA_RUNTIME_DOTNETCORE2)
result = lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
def _fix_next_token_request(data):
# Fix for https://github.com/localstack/localstack/issues/1527
pattern = r'"nextToken":\s*"([0-9]+)"'
replacement = r'"nextToken": \1'
return re.sub(pattern, replacement, to_str(data))
def return_response(self, method, path, data, headers, response):
if path.startswith('/shell') or method == 'GET':
return
data = json.loads(to_str(data))
# update table definitions
if data and 'TableName' in data and 'KeySchema' in data:
TABLE_DEFINITIONS[data['TableName']] = data
if response._content:
# fix the table and latest stream ARNs (DynamoDBLocal hardcodes "ddblocal" as the region)
content_replaced = re.sub(
r'("TableArn"|"LatestStreamArn"|"StreamArn")\s*:\s*"arn:aws:dynamodb:ddblocal:([^"]+)"',
r'\1: "arn:aws:dynamodb:%s:\2"' % aws_stack.get_region(),
to_str(response._content))
if content_replaced != response._content:
response._content = content_replaced
fix_headers_for_updated_response(response)
action = headers.get('X-Amz-Target')
if not action:
return
# upgrade event version to 1.1
record = {
'eventID': '1',
'eventVersion': '1.1',
'dynamodb': {
'ApproximateCreationDateTime': time.time(),
'StreamViewType': 'NEW_AND_OLD_IMAGES',
'SizeBytes': -1
},
'awsRegion': aws_stack.get_region(),
'eventSource': 'aws:dynamodb'
}
records = [record]
streams_enabled_cache = {}
table_name = data.get('TableName')
event_sources_or_streams_enabled = has_event_sources_or_streams_enabled(
table_name, streams_enabled_cache)
if action == '%s.UpdateItem' % ACTION_PREFIX:
if response.status_code == 200 and event_sources_or_streams_enabled:
existing_item = self._thread_local('existing_item')
record[
'eventName'] = 'INSERT' if not existing_item else 'MODIFY'
updated_item = find_existing_item(data)
if not updated_item:
return
record['dynamodb']['Keys'] = data['Key']
if existing_item:
record['dynamodb']['OldImage'] = existing_item
record['dynamodb']['NewImage'] = updated_item
record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item))
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
records = self.prepare_batch_write_item_records(record, data)
for record in records:
event_sources_or_streams_enabled = (
event_sources_or_streams_enabled
or has_event_sources_or_streams_enabled(
record['eventSourceARN'], streams_enabled_cache))
elif action == '%s.TransactWriteItems' % ACTION_PREFIX:
records = self.prepare_transact_write_item_records(record, data)
for record in records:
event_sources_or_streams_enabled = (
event_sources_or_streams_enabled
or has_event_sources_or_streams_enabled(
record['eventSourceARN'], streams_enabled_cache))
elif action == '%s.PutItem' % ACTION_PREFIX:
if response.status_code == 200:
keys = dynamodb_extract_keys(item=data['Item'],
table_name=table_name)
if isinstance(keys, Response):
return keys
# fix response
if response._content == '{}':
response._content = update_put_item_response_content(
data, response._content)
fix_headers_for_updated_response(response)
if event_sources_or_streams_enabled:
existing_item = self._thread_local('existing_item')
record[
'eventName'] = 'INSERT' if not existing_item else 'MODIFY'
# prepare record keys
record['dynamodb']['Keys'] = keys
record['dynamodb']['NewImage'] = data['Item']
record['dynamodb']['SizeBytes'] = len(
json.dumps(data['Item']))
if existing_item:
record['dynamodb']['OldImage'] = existing_item
elif action in [
'%s.GetItem' % ACTION_PREFIX,
'%s.Query' % ACTION_PREFIX
]:
if response.status_code == 200:
content = json.loads(to_str(response.content))
# make sure we append 'ConsumedCapacity', which is properly
# returned by dynalite, but not by AWS's DynamoDBLocal
if 'ConsumedCapacity' not in content and data.get(
'ReturnConsumedCapacity') in ['TOTAL', 'INDEXES']:
content['ConsumedCapacity'] = {
'TableName': table_name,
'CapacityUnits': 5, # TODO hardcoded
'ReadCapacityUnits': 2,
'WriteCapacityUnits': 3
}
response._content = json.dumps(content)
fix_headers_for_updated_response(response)
elif action == '%s.DeleteItem' % ACTION_PREFIX:
if response.status_code == 200 and event_sources_or_streams_enabled:
old_item = self._thread_local('existing_item')
record['eventName'] = 'REMOVE'
record['dynamodb']['Keys'] = data['Key']
record['dynamodb']['OldImage'] = old_item
elif action == '%s.CreateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
if response.status_code == 200:
content = json.loads(to_str(response._content))
create_dynamodb_stream(
data,
content['TableDescription'].get('LatestStreamLabel'))
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_CREATE_TABLE,
payload={'n': event_publisher.get_hash(table_name)})
if data.get('Tags') and response.status_code == 200:
table_arn = json.loads(
response._content)['TableDescription']['TableArn']
TABLE_TAGS[table_arn] = {
tag['Key']: tag['Value']
for tag in data['Tags']
}
return
elif action == '%s.DeleteTable' % ACTION_PREFIX:
if response.status_code == 200:
table_arn = json.loads(response._content).get(
'TableDescription', {}).get('TableArn')
event_publisher.fire_event(
event_publisher.EVENT_DYNAMODB_DELETE_TABLE,
payload={'n': event_publisher.get_hash(table_name)})
self.delete_all_event_source_mappings(table_arn)
dynamodbstreams_api.delete_streams(table_arn)
TABLE_TAGS.pop(table_arn, None)
return
elif action == '%s.UpdateTable' % ACTION_PREFIX:
if 'StreamSpecification' in data:
if response.status_code == 200:
content = json.loads(to_str(response._content))
create_dynamodb_stream(
data,
content['TableDescription'].get('LatestStreamLabel'))
return
elif action == '%s.TagResource' % ACTION_PREFIX:
table_arn = data['ResourceArn']
if table_arn not in TABLE_TAGS:
TABLE_TAGS[table_arn] = {}
TABLE_TAGS[table_arn].update(
{tag['Key']: tag['Value']
for tag in data.get('Tags', [])})
return
elif action == '%s.UntagResource' % ACTION_PREFIX:
table_arn = data['ResourceArn']
for tag_key in data.get('TagKeys', []):
TABLE_TAGS.get(table_arn, {}).pop(tag_key, None)
return
else:
# nothing to do
return
if event_sources_or_streams_enabled and records and 'eventName' in records[
0]:
if 'TableName' in data:
records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn(
table_name)
forward_to_lambda(records)
forward_to_ddb_stream(records)
def _test_api_gateway_lambda_proxy_integration(self, fn_name, path):
self.create_lambda_function(fn_name)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(fn_name)
invocation_uri = 'arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations'
target_uri = invocation_uri % (DEFAULT_REGION, lambda_uri)
result = self.connect_api_gateway_to_http_with_lambda_proxy(
'test_gateway2', target_uri, path=path)
api_id = result['id']
path_map = get_rest_api_paths(api_id)
_, resource = get_resource_for_path('/lambda/foo1', path_map)
# make test request to gateway and check response
path = path.replace('{test_param1}', 'foo1')
path = path + '?foo=foo&bar=bar&bar=baz'
url = INBOUND_GATEWAY_URL_PATTERN.format(
api_id=api_id, stage_name=self.TEST_STAGE_NAME, path=path)
data = {'return_status_code': 203, 'return_headers': {'foo': 'bar123'}}
result = requests.post(
url,
data=json.dumps(data),
headers={'User-Agent': 'python-requests/testing'})
self.assertEqual(result.status_code, 203)
self.assertEqual(result.headers.get('foo'), 'bar123')
parsed_body = json.loads(to_str(result.content))
self.assertEqual(parsed_body.get('return_status_code'), 203)
self.assertDictEqual(parsed_body.get('return_headers'),
{'foo': 'bar123'})
self.assertDictEqual(parsed_body.get('queryStringParameters'), {
'foo': 'foo',
'bar': ['bar', 'baz']
})
request_context = parsed_body.get('requestContext')
source_ip = request_context['identity'].pop('sourceIp')
self.assertTrue(
re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', source_ip))
self.assertEqual(request_context['path'], '/lambda/foo1')
self.assertEqual(request_context['accountId'], TEST_AWS_ACCOUNT_ID)
self.assertEqual(request_context['resourceId'], resource.get('id'))
self.assertEqual(request_context['stage'], self.TEST_STAGE_NAME)
self.assertEqual(request_context['identity']['userAgent'],
'python-requests/testing')
result = requests.delete(url, data=json.dumps(data))
self.assertEqual(result.status_code, 404)
# send message with non-ASCII chars
body_msg = '🙀 - 参よ'
result = requests.post(url,
data=json.dumps({'return_raw_body': body_msg}))
self.assertEqual(to_str(result.content), body_msg)
def forward_request(self, method, path, data, headers):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
req_action = req_data['Action'][0]
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
do_create_topic(topic_arn)
if req_action == 'SetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(message='Unable to find subscription for given ARN', code=400)
attr_name = req_data['AttributeName'][0]
attr_value = req_data['AttributeValue'][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == 'GetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(message='Unable to find subscription for given ARN', code=400)
content = '<Attributes>'
for key, value in sub.items():
content += '<entry><key>%s</key><value>%s</value></entry>\n' % (key, value)
content += '</Attributes>'
return make_response(req_action, content=content)
elif req_action == 'Subscribe':
if 'Endpoint' not in req_data:
return make_error(message='Endpoint not specified in subscription', code=400)
elif req_action == 'Unsubscribe':
if 'SubscriptionArn' not in req_data:
return make_error(message='SubscriptionArn not specified in unsubscribe request', code=400)
do_unsubscribe(req_data.get('SubscriptionArn')[0])
elif req_action == 'Publish':
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS[topic_arn]:
filter_policy = json.loads(subscriber.get('FilterPolicy', '{}'))
message_attributes = get_message_attributes(req_data)
if check_filter_policy(filter_policy, message_attributes):
if subscriber['Protocol'] == 'sqs':
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
try:
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data)
)
except Exception as exc:
return make_error(message=str(exc), code=400)
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn, message, subject=req_data.get('Subject', [None])[0]
)
elif subscriber['Protocol'] in ['http', 'https']:
try:
message_body = create_sns_message_body(subscriber, req_data)
except Exception as exc:
return make_error(message=str(exc), code=400)
requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
'x-amz-sns-message-type': 'Notification'
},
data=message_body
)
else:
LOGGER.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['Protocol'])
# return response here because we do not want the request to be forwarded to SNS
return make_response(req_action)
return True
def download_s3_object(s3, bucket, path):
with tempfile.SpooledTemporaryFile() as tmpfile:
s3.Bucket(bucket).download_fileobj(path, tmpfile)
tmpfile.seek(0)
return to_str(tmpfile.read())
def forward_request(self, method, path, data, headers):
records.append((json.loads(to_str(data)), headers))
return 429
def post_request():
action = request.headers.get('x-amz-target')
data = json.loads(to_str(request.data))
response = None
if action == '%s.ListDeliveryStreams' % ACTION_HEADER_PREFIX:
response = {
'DeliveryStreamNames': get_delivery_stream_names(),
'HasMoreDeliveryStreams': False
}
elif action == '%s.CreateDeliveryStream' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
region_name = extract_region_from_auth_header(request.headers)
response = create_stream(
stream_name,
delivery_stream_type=data.get('DeliveryStreamType'),
delivery_stream_type_configuration=data.get(
'KinesisStreamSourceConfiguration'),
s3_destination=data.get('S3DestinationConfiguration'),
elasticsearch_destination=data.get(
'ElasticsearchDestinationConfiguration'),
tags=data.get('Tags'),
region_name=region_name)
elif action == '%s.DeleteDeliveryStream' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
response = delete_stream(stream_name)
elif action == '%s.DescribeDeliveryStream' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
response = get_stream(stream_name)
if not response:
return error_not_found(stream_name)
response = {'DeliveryStreamDescription': response}
elif action == '%s.PutRecord' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
record = data['Record']
put_record(stream_name, record)
response = {'RecordId': str(uuid.uuid4())}
elif action == '%s.PutRecordBatch' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
records = data['Records']
put_records(stream_name, records)
request_responses = []
for i in records:
request_responses.append({'RecordId': str(uuid.uuid4())})
response = {'FailedPutCount': 0, 'RequestResponses': request_responses}
elif action == '%s.UpdateDestination' % ACTION_HEADER_PREFIX:
stream_name = data['DeliveryStreamName']
version_id = data['CurrentDeliveryStreamVersionId']
destination_id = data['DestinationId']
s3_update = data[
'S3DestinationUpdate'] if 'S3DestinationUpdate' in data else None
update_destination(stream_name=stream_name,
destination_id=destination_id,
s3_update=s3_update,
version_id=version_id)
es_update = data[
'ESDestinationUpdate'] if 'ESDestinationUpdate' in data else None
update_destination(stream_name=stream_name,
destination_id=destination_id,
elasticsearch_update=es_update,
version_id=version_id)
response = {}
elif action == '%s.ListTagsForDeliveryStream' % ACTION_HEADER_PREFIX:
response = get_delivery_stream_tags(data['DeliveryStreamName'],
data.get('ExclusiveStartTagKey'),
data.get('Limit', 50))
else:
response = error_response('Unknown action "%s"' % action,
code=400,
error_type='InvalidAction')
if isinstance(response, dict):
response = jsonify(response)
return response
def _replace(response, pattern, replacement):
content = to_str(response.content)
response._content = re.sub(pattern, replacement, content)
def forward_request(self, method, path, data, headers):
if method == "OPTIONS":
return 200
# check region
try:
aws_stack.check_valid_region(headers)
aws_stack.set_default_region_in_headers(headers)
except Exception as e:
return make_error(message=str(e), code=400)
if method == "POST":
# parse payload and extract fields
req_data = parse_qs(to_str(data), keep_blank_values=True)
# parse data from query path
if not req_data:
parsed_path = urlparse(path)
req_data = parse_qs(parsed_path.query, keep_blank_values=True)
req_action = req_data["Action"][0]
topic_arn = (
req_data.get("TargetArn") or req_data.get("TopicArn") or req_data.get("ResourceArn")
)
if topic_arn:
topic_arn = topic_arn[0]
topic_arn = aws_stack.fix_account_id_in_arns(topic_arn)
if req_action == "SetSubscriptionAttributes":
sub = get_subscription_by_arn(req_data["SubscriptionArn"][0])
if not sub:
return make_error(message="Unable to find subscription for given ARN", code=400)
attr_name = req_data["AttributeName"][0]
attr_value = req_data["AttributeValue"][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == "GetSubscriptionAttributes":
sub = get_subscription_by_arn(req_data["SubscriptionArn"][0])
if not sub:
return make_error(
message="Subscription with arn {0} not found".format(
req_data["SubscriptionArn"][0]
),
code=404,
code_string="NotFound",
)
content = "<Attributes>"
for key, value in sub.items():
if key in HTTP_SUBSCRIPTION_ATTRIBUTES:
continue
content += "<entry><key>%s</key><value>%s</value></entry>\n" % (
key,
value,
)
content += "</Attributes>"
return make_response(req_action, content=content)
elif req_action == "Subscribe":
if "Endpoint" not in req_data:
return make_error(message="Endpoint not specified in subscription", code=400)
if req_data["Protocol"][0] not in SNS_PROTOCOLS:
return make_error(
message=f"Invalid parameter: Amazon SNS does not support this protocol string: "
f"{req_data['Protocol'][0]}",
code=400,
)
if ".fifo" in req_data["Endpoint"][0] and ".fifo" not in topic_arn:
return make_error(
message="FIFO SQS Queues can not be subscribed to standard SNS topics",
code=400,
code_string="InvalidParameter",
)
elif req_action == "ConfirmSubscription":
if "TopicArn" not in req_data:
return make_error(
message="TopicArn not specified in confirm subscription request",
code=400,
)
if "Token" not in req_data:
return make_error(
message="Token not specified in confirm subscription request",
code=400,
)
do_confirm_subscription(req_data.get("TopicArn")[0], req_data.get("Token")[0])
elif req_action == "Unsubscribe":
if "SubscriptionArn" not in req_data:
return make_error(
message="SubscriptionArn not specified in unsubscribe request",
code=400,
)
do_unsubscribe(req_data.get("SubscriptionArn")[0])
elif req_action == "DeleteTopic":
do_delete_topic(topic_arn)
elif req_action == "Publish":
if req_data.get("Subject") == [""]:
return make_error(code=400, code_string="InvalidParameter", message="Subject")
if not req_data.get("Message") or all(
not message for message in req_data.get("Message")
):
return make_error(
code=400, code_string="InvalidParameter", message="Empty message"
)
if topic_arn and ".fifo" in topic_arn and not req_data.get("MessageGroupId"):
return make_error(
code=400,
code_string="InvalidParameter",
message="The MessageGroupId parameter is required for FIFO topics",
)
sns_backend = SNSBackend.get()
# No need to create a topic to send SMS or single push notifications with SNS
# but we can't mock a sending so we only return that it went well
if "PhoneNumber" not in req_data and "TargetArn" not in req_data:
if topic_arn not in sns_backend.sns_subscriptions:
return make_error(
code=404,
code_string="NotFound",
message="Topic does not exist",
)
message_id = publish_message(topic_arn, req_data, headers)
# return response here because we do not want the request to be forwarded to SNS backend
return make_response(req_action, message_id=message_id)
elif req_action == "PublishBatch":
entries = parse_urlencoded_data(
req_data, "PublishBatchRequestEntries.member", "MessageAttributes.entry"
)
if len(entries) > 10:
return make_error(
message="The batch request contains more entries than permissible",
code=400,
code_string="TooManyEntriesInBatchRequest",
)
ids = [entry["Id"] for entry in entries]
if len(set(ids)) != len(entries):
return make_error(
message="Two or more batch entries in the request have the same Id",
code=400,
code_string="BatchEntryIdsNotDistinct",
)
if topic_arn and ".fifo" in topic_arn:
if not all(["MessageGroupId" in entry for entry in entries]):
return make_error(
message="The MessageGroupId parameter is required for FIFO topics",
code=400,
code_string="InvalidParameter",
)
response = publish_batch(topic_arn, entries, headers)
return requests_response_xml(
req_action, response, xmlns="http://sns.amazonaws.com/doc/2010-03-31/"
)
elif req_action == "ListTagsForResource":
tags = do_list_tags_for_resource(topic_arn)
content = "<Tags/>"
if len(tags) > 0:
content = "<Tags>"
for tag in tags:
content += "<member>"
content += "<Key>%s</Key>" % tag["Key"]
content += "<Value>%s</Value>" % tag["Value"]
content += "</member>"
content += "</Tags>"
return make_response(req_action, content=content)
elif req_action == "CreateTopic":
sns_backend = SNSBackend.get()
topic_arn = aws_stack.sns_topic_arn(req_data["Name"][0])
tag_resource_success = self._extract_tags(topic_arn, req_data, True, sns_backend)
sns_backend.sns_subscriptions[topic_arn] = (
sns_backend.sns_subscriptions.get(topic_arn) or []
)
# in case if there is an error it returns an error , other wise it will continue as expected.
if not tag_resource_success:
return make_error(
code=400,
code_string="InvalidParameter",
message="Topic already exists with different tags",
)
elif req_action == "TagResource":
sns_backend = SNSBackend.get()
self._extract_tags(topic_arn, req_data, False, sns_backend)
return make_response(req_action)
elif req_action == "UntagResource":
tags_to_remove = []
req_tags = {k: v for k, v in req_data.items() if k.startswith("TagKeys.member.")}
req_tags = req_tags.values()
for tag in req_tags:
tags_to_remove.append(tag[0])
do_untag_resource(topic_arn, tags_to_remove)
return make_response(req_action)
data = self._reset_account_id(data)
return Request(data=data, headers=headers, method=method)
return True
def _fix_next_token_response(response):
# Fix for https://github.com/localstack/localstack/issues/1527
pattern = r'"nextToken":\s*([0-9]+)'
replacement = r'"nextToken": "\1"'
response._content = re.sub(pattern, replacement,
to_str(response.content))
raise Exception('Not implemented.')
def startup(self):
pass
def cleanup(self, arn=None):
pass
def run_lambda_executor(self, cmd, env_vars={}, async=False):
process = run(cmd, async=True, stderr=subprocess.PIPE, outfile=subprocess.PIPE, env_vars=env_vars)
if async:
result = '{"async": "%s"}' % async
log_output = 'Lambda executed asynchronously'
else:
return_code = process.wait()
result = to_str(process.stdout.read())
log_output = to_str(process.stderr.read())
if return_code != 0:
raise Exception('Lambda process returned error status code: %s. Output:\n%s' %
(return_code, log_output))
return result, log_output
# holds information about an existing container.
class ContainerInfo:
"""
Contains basic information about a docker container.
"""
def __init__(self, name, entry_point):
self.name = name
def invoke_function(function):
""" Invoke an existing function
---
operationId: 'invokeFunction'
parameters:
- name: 'request'
in: body
"""
# function here can either be an arn or a function name
arn = func_arn(function)
# arn can also contain a qualifier, extract it from there if so
m = re.match('(arn:aws:lambda:.*:.*:function:[a-zA-Z0-9-_]+)(:.*)?', arn)
if m and m.group(2):
qualifier = m.group(2)[1:]
arn = m.group(1)
else:
qualifier = request.args.get('Qualifier')
data = request.get_data()
if data:
data = to_str(data)
try:
data = json.loads(data)
except Exception:
try:
# try to read chunked content
data = json.loads(parse_chunked_data(data))
except Exception:
return error_response(
'The payload is not JSON: %s' % data,
415,
error_type='UnsupportedMediaTypeException')
# Default invocation type is RequestResponse
invocation_type = request.environ.get('HTTP_X_AMZ_INVOCATION_TYPE',
'RequestResponse')
def _create_response(result, status_code=200):
""" Create the final response for the given invocation result """
if isinstance(result, Response):
return result
details = {'StatusCode': status_code, 'Payload': result, 'Headers': {}}
if isinstance(result, dict):
for key in ('StatusCode', 'Payload', 'FunctionError'):
if result.get(key):
details[key] = result[key]
# Try to parse parse payload as JSON
payload = details['Payload']
if payload and isinstance(
payload, (str, bytes)) and payload[0] in ('[', '{', '"'):
try:
details['Payload'] = json.loads(details['Payload'])
except Exception:
pass
# Set error headers
if details.get('FunctionError'):
details['Headers']['X-Amz-Function-Error'] = str(
details['FunctionError'])
# Construct response object
response_obj = details['Payload']
if isinstance(response_obj,
(dict, list, bool)) or is_number(response_obj):
# Assume this is a JSON response
response_obj = jsonify(response_obj)
else:
response_obj = str(response_obj)
details['Headers']['Content-Type'] = 'text/plain'
return response_obj, details['StatusCode'], details['Headers']
# check if this lambda function exists
not_found = None
if arn not in arn_to_lambda:
not_found = not_found_error(arn)
elif qualifier and not arn_to_lambda.get(arn).qualifier_exists(qualifier):
not_found = not_found_error('{0}:{1}'.format(arn, qualifier))
if not_found:
forward_result = forward_to_fallback_url(func_arn, data)
if forward_result is not None:
return _create_response(forward_result)
return not_found
if invocation_type == 'RequestResponse':
result = run_lambda(asynchronous=False,
func_arn=arn,
event=data,
context={},
version=qualifier)
return _create_response(result)
elif invocation_type == 'Event':
run_lambda(asynchronous=True,
func_arn=arn,
event=data,
context={},
version=qualifier)
return _create_response('', status_code=202)
elif invocation_type == 'DryRun':
# Assume the dry run always passes.
return _create_response('', status_code=204)
return error_response(
'Invocation type not one of: RequestResponse, Event or DryRun',
code=400,
error_type='InvalidParameterValueException')
def test_lambda_runtimes():
lambda_client = aws_stack.connect_to_service('lambda')
# deploy and invoke lambda - Python 2.7
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY,
zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
if use_docker():
# deploy and invoke lambda - Python 3.6
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON3), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3,
zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke lambda - Java
if not os.path.exists(TEST_LAMBDA_JAVA):
mkdir(os.path.dirname(TEST_LAMBDA_JAVA))
download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA)
zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA, zip_file=zip_file,
runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert 'LinkedHashMap' in to_str(result_data)
# test SNSEvent
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event',
Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {'async': 'True'}
# test DDBEvent
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event',
Payload=b'{"Records": [{"dynamodb": {"Message": "{}"}}]}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {'async': 'True'}
# test KinesisEvent
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA,
Payload=b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert 'KinesisEvent' in to_str(result_data)
# deploy and invoke lambda - Java with stream handler
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=zip_file,
runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke lambda - Java with serializable input object
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, zip_file=zip_file,
runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.SerializedInputLambdaHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE,
Payload=b'{"bucket": "test_bucket", "key": "test_key"}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {'validated': True, 'bucket': 'test_bucket', 'key': 'test_key'}
if use_docker():
# deploy and invoke lambda - Node.js
zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS,
zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke - .NET Core 2.0. Its already a zip
zip_file = TEST_LAMBDA_DOTNETCORE2
zip_file_content = None
with open(zip_file, 'rb') as file_obj:
zip_file_content = file_obj.read()
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DOTNETCORE2, zip_file=zip_file_content,
handler='DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler',
runtime=LAMBDA_RUNTIME_DOTNETCORE2)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
def return_response(self, method, path, data, headers, response):
parsed = urlparse.urlparse(path)
# TODO: consider the case of hostname-based (as opposed to path-based) bucket addressing
bucket_name = parsed.path.split('/')[1]
# POST requests to S3 may include a success_action_redirect field,
# which should be used to redirect a client to a new location.
if method == 'POST':
key, redirect_url = multipart_content.find_multipart_redirect_url(
data, headers)
if key and redirect_url:
response.status_code = 303
response.headers['Location'] = expand_redirect_url(
redirect_url, key, bucket_name)
LOGGER.debug('S3 POST {} to {}'.format(
response.status_code, response.headers['Location']))
# get subscribers and send bucket notifications
if method in ('PUT', 'DELETE') and '/' in path[1:]:
# check if this is an actual put object request, because it could also be
# a put bucket request with a path like this: /bucket_name/
if len(path[1:].split('/')[1]) > 0:
parts = parsed.path[1:].split('/', 1)
# ignore bucket notification configuration requests
if parsed.query != 'notification' and parsed.query != 'lifecycle':
object_path = parts[1] if parts[1][
0] == '/' else '/%s' % parts[1]
send_notifications(method, bucket_name, object_path)
# publish event for creation/deletion of buckets:
if method in ('PUT', 'DELETE') and ('/' not in path[1:] or
len(path[1:].split('/')[1]) <= 0):
event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method
== 'PUT' else event_publisher.EVENT_S3_DELETE_BUCKET)
event_publisher.fire_event(
event_type,
payload={'n': event_publisher.get_hash(bucket_name)})
# fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382)
if method == 'PUT' and parsed.query == 'policy':
response._content = ''
response.status_code = 204
return response
# append CORS headers to response
if response:
append_cors_headers(bucket_name,
request_method=method,
request_headers=headers,
response=response)
response_content_str = None
try:
response_content_str = to_str(response._content)
except Exception:
pass
# we need to un-pretty-print the XML, otherwise we run into this issue with Spark:
# https://github.com/jserver/mock-s3/pull/9/files
# https://github.com/localstack/localstack/issues/183
# Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n
if response_content_str and response_content_str.startswith('<'):
is_bytes = isinstance(response._content, six.binary_type)
response._content = re.sub(r'([^\?])>\n\s*<',
r'\1><',
response_content_str,
flags=re.MULTILINE)
if is_bytes:
response._content = to_bytes(response._content)
# update content-length headers (fix https://github.com/localstack/localstack/issues/541)
if isinstance(response._content,
(six.string_types, six.binary_type)):
response.headers['content-length'] = len(response._content)
def forward_request(self, method, path, data, headers):
result = handle_special_request(method, path, data, headers)
if result is not None:
return result
if not data:
data = '{}'
data = json.loads(to_str(data))
ddb_client = aws_stack.connect_to_service('dynamodb')
action = headers.get('X-Amz-Target')
if self.should_throttle(action):
return error_response_throughput()
ProxyListenerDynamoDB.thread_local.existing_item = None
if action == '%s.CreateTable' % ACTION_PREFIX:
# Check if table exists, to avoid error log output from DynamoDBLocal
if self.table_exists(ddb_client, data['TableName']):
return error_response(message='Table already created',
error_type='ResourceInUseException',
code=400)
if action == '%s.CreateGlobalTable' % ACTION_PREFIX:
return create_global_table(data)
elif action == '%s.DescribeGlobalTable' % ACTION_PREFIX:
return describe_global_table(data)
elif action == '%s.ListGlobalTables' % ACTION_PREFIX:
return list_global_tables(data)
elif action == '%s.UpdateGlobalTable' % ACTION_PREFIX:
return update_global_table(data)
elif action in ('%s.PutItem' % ACTION_PREFIX,
'%s.UpdateItem' % ACTION_PREFIX,
'%s.DeleteItem' % ACTION_PREFIX):
# find an existing item and store it in a thread-local, so we can access it in return_response,
# in order to determine whether an item already existed (MODIFY) or not (INSERT)
try:
if has_event_sources_or_streams_enabled(data['TableName']):
ProxyListenerDynamoDB.thread_local.existing_item = find_existing_item(
data)
except Exception as e:
if 'ResourceNotFoundException' in str(e):
return get_table_not_found_error()
raise
# Fix incorrect values if ReturnValues==ALL_OLD and ReturnConsumedCapacity is
# empty, see https://github.com/localstack/localstack/issues/2049
if ((data.get('ReturnValues') == 'ALL_OLD') or (not data.get('ReturnValues'))) \
and not data.get('ReturnConsumedCapacity'):
data['ReturnConsumedCapacity'] = 'TOTAL'
return Request(data=json.dumps(data),
method=method,
headers=headers)
elif action == '%s.DescribeTable' % ACTION_PREFIX:
# Check if table exists, to avoid error log output from DynamoDBLocal
if not self.table_exists(ddb_client, data['TableName']):
return get_table_not_found_error()
elif action == '%s.DeleteTable' % ACTION_PREFIX:
# Check if table exists, to avoid error log output from DynamoDBLocal
if not self.table_exists(ddb_client, data['TableName']):
return get_table_not_found_error()
elif action == '%s.BatchWriteItem' % ACTION_PREFIX:
existing_items = []
for table_name in sorted(data['RequestItems'].keys()):
for request in data['RequestItems'][table_name]:
for key in ['PutRequest', 'DeleteRequest']:
inner_request = request.get(key)
if inner_request:
existing_items.append(
find_existing_item(inner_request, table_name))
ProxyListenerDynamoDB.thread_local.existing_items = existing_items
elif action == '%s.Query' % ACTION_PREFIX:
if data.get('IndexName'):
if not is_index_query_valid(to_str(data['TableName']),
data.get('Select')):
return error_response(
message=
'One or more parameter values were invalid: Select type '
'ALL_ATTRIBUTES is not supported for global secondary index id-index '
'because its projection type is not ALL',
error_type='ValidationException',
code=400)
elif action == '%s.TransactWriteItems' % ACTION_PREFIX:
existing_items = []
for item in data['TransactItems']:
for key in ['Put', 'Update', 'Delete']:
inner_item = item.get(key)
if inner_item:
existing_items.append(find_existing_item(inner_item))
ProxyListenerDynamoDB.thread_local.existing_items = existing_items
elif action == '%s.UpdateTimeToLive' % ACTION_PREFIX:
# TODO: TTL status is maintained/mocked but no real expiry is happening for items
response = Response()
response.status_code = 200
self._table_ttl_map[data['TableName']] = {
'AttributeName':
data['TimeToLiveSpecification']['AttributeName'],
'Status': data['TimeToLiveSpecification']['Enabled']
}
response._content = json.dumps(
{'TimeToLiveSpecification': data['TimeToLiveSpecification']})
fix_headers_for_updated_response(response)
return response
elif action == '%s.DescribeTimeToLive' % ACTION_PREFIX:
response = Response()
response.status_code = 200
if data['TableName'] in self._table_ttl_map:
if self._table_ttl_map[data['TableName']]['Status']:
ttl_status = 'ENABLED'
else:
ttl_status = 'DISABLED'
response._content = json.dumps({
'TimeToLiveDescription': {
'AttributeName':
self._table_ttl_map[data['TableName']]
['AttributeName'],
'TimeToLiveStatus':
ttl_status
}
})
else: # TTL for dynamodb table not set
response._content = json.dumps({
'TimeToLiveDescription': {
'TimeToLiveStatus': 'DISABLED'
}
})
fix_headers_for_updated_response(response)
return response
elif action == '%s.TagResource' % ACTION_PREFIX or action == '%s.UntagResource' % ACTION_PREFIX:
response = Response()
response.status_code = 200
response._content = '' # returns an empty body on success.
fix_headers_for_updated_response(response)
return response
elif action == '%s.ListTagsOfResource' % ACTION_PREFIX:
response = Response()
response.status_code = 200
response._content = json.dumps({
'Tags': [{
'Key': k,
'Value': v
} for k, v in TABLE_TAGS.get(data['ResourceArn'], {}).items()]
})
fix_headers_for_updated_response(response)
return response
return True
def fix_hardcoded_creation_date(response):
search = '<CreationTime>2011-05-23T15:47:44Z</CreationTime>'
replace = '<CreationTime>%s</CreationTime>' % timestamp()
response._content = to_str(response._content
or '').replace(search, replace)
response.headers['Content-Length'] = str(len(response._content))
def forward_request(self, method, path, data, headers):
if method == 'OPTIONS':
return 200
# check region
try:
aws_stack.check_valid_region(headers)
aws_stack.set_default_region_in_headers(headers)
except Exception as e:
return make_error(message=str(e), code=400)
if method == 'POST':
# parse payload and extract fields
req_data = urlparse.parse_qs(to_str(data), keep_blank_values=True)
# parse data from query path
if not req_data:
parsed_path = urlparse.urlparse(path)
req_data = urlparse.parse_qs(parsed_path.query,
keep_blank_values=True)
req_action = req_data['Action'][0]
topic_arn = req_data.get('TargetArn') or req_data.get(
'TopicArn') or req_data.get('ResourceArn')
if topic_arn:
topic_arn = topic_arn[0]
topic_arn = aws_stack.fix_account_id_in_arns(topic_arn)
if req_action == 'SetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
attr_name = req_data['AttributeName'][0]
attr_value = req_data['AttributeValue'][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == 'GetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
content = '<Attributes>'
for key, value in sub.items():
content += '<entry><key>%s</key><value>%s</value></entry>\n' % (
key, value)
content += '</Attributes>'
return make_response(req_action, content=content)
elif req_action == 'Subscribe':
if 'Endpoint' not in req_data:
return make_error(
message='Endpoint not specified in subscription',
code=400)
elif req_action == 'ConfirmSubscription':
if 'TopicArn' not in req_data:
return make_error(
message=
'TopicArn not specified in confirm subscription request',
code=400)
if 'Token' not in req_data:
return make_error(
message=
'Token not specified in confirm subscription request',
code=400)
do_confirm_subscription(
req_data.get('TopicArn')[0],
req_data.get('Token')[0])
elif req_action == 'Unsubscribe':
if 'SubscriptionArn' not in req_data:
return make_error(
message=
'SubscriptionArn not specified in unsubscribe request',
code=400)
do_unsubscribe(req_data.get('SubscriptionArn')[0])
elif req_action == 'DeleteTopic':
do_delete_topic(topic_arn)
elif req_action == 'Publish':
if req_data.get('Subject') == ['']:
return make_error(code=400,
code_string='InvalidParameter',
message='Subject')
# No need to create a topic to send SMS or single push notifications with SNS
# but we can't mock a sending so we only return that it went well
if 'PhoneNumber' not in req_data and 'TargetArn' not in req_data:
if topic_arn not in SNS_SUBSCRIPTIONS.keys():
return make_error(code=404,
code_string='NotFound',
message='Topic does not exist')
publish_message(topic_arn, req_data)
# return response here because we do not want the request to be forwarded to SNS backend
return make_response(req_action)
elif req_action == 'ListTagsForResource':
tags = do_list_tags_for_resource(topic_arn)
content = '<Tags/>'
if len(tags) > 0:
content = '<Tags>'
for tag in tags:
content += '<member>'
content += '<Key>%s</Key>' % tag['Key']
content += '<Value>%s</Value>' % tag['Value']
content += '</member>'
content += '</Tags>'
return make_response(req_action, content=content)
elif req_action == 'CreateTopic':
topic_arn = aws_stack.sns_topic_arn(req_data['Name'][0])
tag_resource_success = self._extract_tags(
topic_arn, req_data, True)
# in case if there is an error it returns an error , other wise it will continue as expected.
if not tag_resource_success:
return make_error(
code=400,
code_string='InvalidParameter',
message='Topic already exists with different tags')
elif req_action == 'TagResource':
self._extract_tags(topic_arn, req_data, False)
return make_response(req_action)
elif req_action == 'UntagResource':
tags_to_remove = []
req_tags = {
k: v
for k, v in req_data.items()
if k.startswith('TagKeys.member.')
}
req_tags = req_tags.values()
for tag in req_tags:
tags_to_remove.append(tag[0])
do_untag_resource(topic_arn, tags_to_remove)
return make_response(req_action)
data = self._reset_account_id(data)
return Request(data=data, headers=headers, method=method)
return True
def return_response(self, method, path, data, headers, response):
bucket_name = get_bucket_name(path, headers)
# No path-name based bucket name? Try host-based
hostname_parts = headers['host'].split('.')
if (not bucket_name or len(bucket_name) == 0) and len(hostname_parts) > 1:
bucket_name = hostname_parts[0]
# POST requests to S3 may include a success_action_redirect field,
# which should be used to redirect a client to a new location.
key = None
if method == 'POST':
key, redirect_url = multipart_content.find_multipart_redirect_url(data, headers)
if key and redirect_url:
response.status_code = 303
response.headers['Location'] = expand_redirect_url(redirect_url, key, bucket_name)
LOGGER.debug('S3 POST {} to {}'.format(response.status_code, response.headers['Location']))
parsed = urlparse.urlparse(path)
bucket_name_in_host = headers['host'].startswith(bucket_name)
should_send_notifications = all([
method in ('PUT', 'POST', 'DELETE'),
'/' in path[1:] or bucket_name_in_host,
# check if this is an actual put object request, because it could also be
# a put bucket request with a path like this: /bucket_name/
bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0),
self.is_query_allowable(method, parsed.query)
])
# get subscribers and send bucket notifications
if should_send_notifications:
# if we already have a good key, use it, otherwise examine the path
if key:
object_path = '/' + key
elif bucket_name_in_host:
object_path = parsed.path
else:
parts = parsed.path[1:].split('/', 1)
object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1]
send_notifications(method, bucket_name, object_path)
# publish event for creation/deletion of buckets:
if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0):
event_type = (event_publisher.EVENT_S3_CREATE_BUCKET if method == 'PUT'
else event_publisher.EVENT_S3_DELETE_BUCKET)
event_publisher.fire_event(event_type, payload={'n': event_publisher.get_hash(bucket_name)})
# fix an upstream issue in moto S3 (see https://github.com/localstack/localstack/issues/382)
if method == 'PUT' and parsed.query == 'policy':
response._content = ''
response.status_code = 204
return response
if response:
# append CORS headers to response
append_cors_headers(bucket_name, request_method=method, request_headers=headers, response=response)
response_content_str = None
try:
response_content_str = to_str(response._content)
except Exception:
pass
# we need to un-pretty-print the XML, otherwise we run into this issue with Spark:
# https://github.com/jserver/mock-s3/pull/9/files
# https://github.com/localstack/localstack/issues/183
# Note: yet, we need to make sure we have a newline after the first line: <?xml ...>\n
if response_content_str and response_content_str.startswith('<'):
is_bytes = isinstance(response._content, six.binary_type)
response._content = re.sub(r'([^\?])>\n\s*<', r'\1><', response_content_str, flags=re.MULTILINE)
if is_bytes:
response._content = to_bytes(response._content)
# fix content-type: https://github.com/localstack/localstack/issues/618
# https://github.com/localstack/localstack/issues/549
if 'text/html' in response.headers.get('Content-Type', ''):
response.headers['Content-Type'] = 'application/xml; charset=utf-8'
response.headers['content-length'] = len(response._content)
# update content-length headers (fix https://github.com/localstack/localstack/issues/541)
if method == 'DELETE':
response.headers['content-length'] = len(response._content)
def _execute(self, func_arn, func_details, event, context=None, version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = self._prepare_environment(func_details)
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
main_endpoint = get_main_endpoint_from_container()
environment['LOCALSTACK_HOSTNAME'] = main_endpoint
environment['_HANDLER'] = handler
if os.environ.get('HTTP_PROXY'):
environment['HTTP_PROXY'] = os.environ['HTTP_PROXY']
if func_details.timeout:
environment['AWS_LAMBDA_FUNCTION_TIMEOUT'] = str(func_details.timeout)
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
environment['AWS_LAMBDA_COGNITO_IDENTITY'] = json.dumps(context.cognito_identity or {})
if context.client_context is not None:
environment['AWS_LAMBDA_CLIENT_CONTEXT'] = json.dumps(to_str(
base64.b64decode(to_bytes(context.client_context))))
# custom command to execute in the container
command = ''
events_file = ''
if USE_CUSTOM_JAVA_EXECUTOR and is_java_lambda(runtime):
# if running a Java Lambda with our custom executor, set up classpath arguments
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
events_file = '_lambda.events.%s.json' % short_uid()
save_file(os.path.join(lambda_cwd, events_file), event_body)
classpath = Util.get_java_classpath(target_file)
command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" %
(taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, events_file))
# accept any self-signed certificates for outgoing calls from the Lambda
if is_nodejs_runtime(runtime):
environment['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
LOG.info('Running lambda cmd: %s' % cmd)
result = self.run_lambda_executor(cmd, stdin, env_vars=environment, func_details=func_details)
# clean up events file
events_file and os.path.exists(events_file) and rm_rf(events_file)
return result
def response_regex_replace(response, search, replace):
response._content = re.sub(search, replace, to_str(response._content), flags=re.DOTALL | re.MULTILINE)
response.headers['Content-Length'] = str(len(response._content))
