def _revert_virtual_host_style(self, request: HttpRequest):
# extract the bucket name from the host part of the request
bucket_name = request.host.split(".")[0]
# split the url and put the bucket name at the front
parts = urlsplit(request.url)
path_parts = parts.path.split("/")
path_parts = [bucket_name] + path_parts
path_parts = [part for part in path_parts if part]
path = "/" + "/".join(path_parts) or "/"
# set the path with the bucket name in the front at the request
# TODO directly modifying the request can cause issues with our handler chain, instead clone the HTTP request
request.path = path
request.raw_path = path
def test_query_parser_non_flattened_list_structure_changed_name():
"""Simple test with a non-flattened list structure where the name of the list differs from the shape's name
(CloudWatch PutMetricData)."""
parser = QueryRequestParser(load_service("cloudwatch"))
request = HttpRequest(
body=to_bytes(
"Action=PutMetricData&"
"Version=2010-08-01&"
"Namespace=TestNamespace&"
"MetricData.member.1.MetricName=buffers&"
"MetricData.member.1.Unit=Bytes&"
"MetricData.member.1.Value=231434333&"
"MetricData.member.1.Dimensions.member.1.Name=InstanceType&"
"MetricData.member.1.Dimensions.member.1.Value=m1.small&"
"AUTHPARAMS"
),
method="POST",
headers={},
path="",
)
operation, params = parser.parse(request)
assert operation.name == "PutMetricData"
assert params == {
"MetricData": [
{
"Dimensions": [{"Name": "InstanceType", "Value": "m1.small"}],
"MetricName": "buffers",
"Unit": "Bytes",
"Value": 231434333.0,
}
],
"Namespace": "TestNamespace",
}
def test_query_parser_flattened_list_structure():
"""Simple test with a flattened list of structures."""
parser = QueryRequestParser(load_service("sqs"))
request = HttpRequest(
body=to_bytes(
"Action=DeleteMessageBatch&"
"Version=2012-11-05&"
"QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&"
"DeleteMessageBatchRequestEntry.1.Id=bar&"
"DeleteMessageBatchRequestEntry.1.ReceiptHandle=foo&"
"DeleteMessageBatchRequestEntry.2.Id=bar&"
"DeleteMessageBatchRequestEntry.2.ReceiptHandle=foo"),
method="POST",
headers={},
path="",
)
operation, params = parser.parse(request)
assert operation.name == "DeleteMessageBatch"
assert params == {
"QueueUrl":
"http://localhost:4566/000000000000/tf-acc-test-queue",
"Entries": [{
"Id": "bar",
"ReceiptHandle": "foo"
}, {
"Id": "bar",
"ReceiptHandle": "foo"
}],
}
def test_dispatch_missing_method_returns_internal_failure():
table: DispatchTable = {}
sqs_service = load_service("sqs")
skeleton = Skeleton(sqs_service, table)
context = RequestContext()
context.account = "test"
context.region = "us-west-1"
context.service = sqs_service
context.request = HttpRequest(
**{
"method": "POST",
"path": "/",
"body":
"Action=DeleteQueue&Version=2012-11-05&QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue",
"headers": _get_sqs_request_headers(),
})
result = skeleton.invoke(context)
# Use the parser from botocore to parse the serialized response
response_parser = create_parser(sqs_service.protocol)
parsed_response = response_parser.parse(
result.to_readonly_response_dict(),
sqs_service.operation_model("SendMessage").output_shape)
assert "Error" in parsed_response
assert parsed_response["Error"] == {
"Code":
"InternalFailure",
"Message":
"API action 'DeleteQueue' for service 'sqs' not yet implemented",
}
def test_dispatch_common_service_exception():
def delete_queue(_context: RequestContext, _request: ServiceRequest):
raise CommonServiceException("NonExistentQueue", "No such queue")
table: DispatchTable = {}
table["DeleteQueue"] = delete_queue
sqs_service = load_service("sqs")
skeleton = Skeleton(sqs_service, table)
context = RequestContext()
context.account = "test"
context.region = "us-west-1"
context.service = sqs_service
context.request = HttpRequest(
**{
"method": "POST",
"path": "/",
"body":
"Action=DeleteQueue&Version=2012-11-05&QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue",
"headers": _get_sqs_request_headers(),
})
result = skeleton.invoke(context)
# Use the parser from botocore to parse the serialized response
response_parser = create_parser(sqs_service.protocol)
parsed_response = response_parser.parse(
result.to_readonly_response_dict(),
sqs_service.operation_model("SendMessage").output_shape)
assert "Error" in parsed_response
assert parsed_response["Error"] == {
"Code": "NonExistentQueue",
"Message": "No such queue",
}
def _botocore_parser_integration_test(service: str,
action: str,
method: str = None,
request_uri: str = None,
headers: dict = None,
expected: dict = None,
**kwargs):
# Load the appropriate service
service = load_service(service)
# Use the serializer from botocore to serialize the request params
serializer = create_serializer(service.protocol)
serialized_request = serializer.serialize_to_request(
kwargs, service.operation_model(action))
body = serialized_request["body"]
if service.protocol in ["query", "ec2"]:
# Serialize the body as query parameter
body = urlencode(serialized_request["body"])
# Use our parser to parse the serialized body
parser = create_parser(service)
operation_model, parsed_request = parser.parse(
HttpRequest(method=method or "GET",
path=request_uri or "",
headers=headers,
body=body))
# Check if the result is equal to the given "expected" dict or the kwargs (if "expected" has not been set)
assert parsed_request == (expected or kwargs)
def parse(self, request: HttpRequest) -> Tuple[OperationModel, Any]:
body = request.get_data(as_text=True)
instance = parse_qs(body, keep_blank_values=True)
if not instance:
# if the body does not contain any information, fallback to the actual query parameters
instance = request.args
# The query parsing returns a list for each entry in the dict (this is how HTTP handles lists in query params).
# However, the AWS Query format does not have any duplicates.
# Therefore we take the first element of each entry in the dict.
instance = {k: self._get_first(v) for k, v in instance.items()}
if "Action" not in instance:
raise ProtocolParserError(
f"Operation detection failed. "
f"Missing Action in request for query-protocol service {self.service}."
)
action = instance["Action"]
try:
operation: OperationModel = self.service.operation_model(action)
except OperationNotFoundError as e:
raise OperationNotFoundParserError(
f"Operation detection failed."
f"Operation {action} could not be found for service {self.service}."
) from e
# There are no uri params in the query protocol (all ops are POST on "/")
uri_params = {}
input_shape: StructureShape = operation.input_shape
parsed = self._parse_shape(request, input_shape, instance, uri_params)
if parsed is None:
return operation, {}
return operation, parsed
def test_query_parser_non_flattened_list_structure():
"""Simple test with a non-flattened list structure (CloudFormation CreateChangeSet)."""
parser = QueryRequestParser(load_service("cloudformation"))
request = HttpRequest(
body=to_bytes(
"Action=CreateChangeSet&"
"ChangeSetName=SampleChangeSet&"
"Parameters.member.1.ParameterKey=KeyName&"
"Parameters.member.1.UsePreviousValue=true&"
"Parameters.member.2.ParameterKey=Purpose&"
"Parameters.member.2.ParameterValue=production&"
"StackName=arn:aws:cloudformation:us-east-1:123456789012:stack/SampleStack/1a2345b6-0000-00a0-a123-00abc0abc000&"
"UsePreviousTemplate=true&"
"Version=2010-05-15&"
"X-Amz-Algorithm=AWS4-HMAC-SHA256&"
"X-Amz-Credential=[Access-key-ID-and-scope]&"
"X-Amz-Date=20160316T233349Z&"
"X-Amz-SignedHeaders=content-type;host&"
"X-Amz-Signature=[Signature]"
),
method="POST",
headers={},
path="",
)
operation, params = parser.parse(request)
assert operation.name == "CreateChangeSet"
assert params == {
"StackName": "arn:aws:cloudformation:us-east-1:123456789012:stack/SampleStack/1a2345b6-0000-00a0-a123-00abc0abc000",
"UsePreviousTemplate": True,
"Parameters": [
{"ParameterKey": "KeyName", "UsePreviousValue": True},
{"ParameterKey": "Purpose", "ParameterValue": "production"},
],
"ChangeSetName": "SampleChangeSet",
}
def test_s3_virtual_host_addressing():
"""Test the parsing of a map with the location trait 'headers'."""
request = HttpRequest(
method="PUT", headers={"host": s3_utils.get_bucket_hostname("test-bucket")}
)
parser = create_parser(load_service("s3"))
parsed_operation_model, parsed_request = parser.parse(request)
assert parsed_operation_model.name == "CreateBucket"
assert "Bucket" in parsed_request
assert parsed_request["Bucket"] == "test-bucket"
def forward_request(self, method, path, data, headers):
request = HttpRequest(
method=method,
path=path,
headers=headers,
body=data,
)
context = self.create_request_context(request)
response = self.skeleton.invoke(context)
return self.to_server_response(response)
def test_restjson_parser_path_params_with_slashes():
parser = RestJSONRequestParser(load_service("qldb"))
resource_arn = "arn:aws:qldb:eu-central-1:000000000000:ledger/c-c67c827a"
request = HttpRequest(
body=b"",
method="GET",
headers={},
path=f"/tags/{resource_arn}",
)
operation, params = parser.parse(request)
assert operation.name == "ListTagsForResource"
assert params == {"ResourceArn": resource_arn}
def parse(self, request: HttpRequest) -> Tuple[OperationModel, Any]:
body = request.get_data(as_text=True)
instance = parse_qs(body, keep_blank_values=True)
# The query parsing returns a list for each entry in the dict (this is how HTTP handles lists in query params).
# However, the AWS Query format does not have any duplicates.
# Therefore we take the first element of each entry in the dict.
instance = {k: self._get_first(v) for k, v in instance.items()}
operation: OperationModel = self.service.operation_model(
instance["Action"])
input_shape: StructureShape = operation.input_shape
return operation, self._parse_shape(request, input_shape, instance)
def forward_request(self, method, path, data, headers):
split_url = urlsplit(path)
request = HttpRequest(
method=method,
path=split_url.path,
query_string=split_url.query,
headers=headers,
body=data,
)
context = self.create_request_context(request)
response = self.skeleton.invoke(context)
return self.to_server_response(response)
def _parse_body_as_json(self, request: HttpRequest) -> dict:
body_contents = request.data
if not body_contents:
return {}
if request.mimetype.startswith("application/x-amz-cbor"):
try:
return cbor2.loads(body_contents)
except ValueError as e:
raise ProtocolParserError("HTTP body could not be parsed as CBOR.") from e
else:
try:
return request.get_json(force=True)
except BadRequest as e:
raise ProtocolParserError("HTTP body could not be parsed as JSON.") from e
def test_parser_error_on_protocol_error():
"""Test that the parser raises a ProtocolParserError in case of invalid data to parse."""
parser = QueryRequestParser(load_service("sqs"))
request = HttpRequest(
body=to_bytes(
"Action=UnknownOperation&Version=2012-11-05&"
"QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&"
"MessageBody=%7B%22foo%22%3A+%22bared%22%7D&"
"DelaySeconds=2"
),
method="POST",
headers={},
path="",
)
with pytest.raises(ProtocolParserError):
parser.parse(request)
def _botocore_parser_integration_test(
service: str, action: str, headers: dict = None, expected: dict = None, **kwargs
):
# Load the appropriate service
service = load_service(service)
# Use the serializer from botocore to serialize the request params
serializer = create_serializer(service.protocol)
operation_model = service.operation_model(action)
serialized_request = serializer.serialize_to_request(kwargs, operation_model)
prepare_request_dict(serialized_request, "")
split_url = urlsplit(serialized_request.get("url"))
path = split_url.path
query_string = split_url.query
body = serialized_request["body"]
# use custom headers (if provided), or headers from serialized request as default
headers = serialized_request.get("headers") if headers is None else headers
if service.protocol in ["query", "ec2"]:
# Serialize the body as query parameter
body = urlencode(serialized_request["body"])
# Use our parser to parse the serialized body
parser = create_parser(service)
parsed_operation_model, parsed_request = parser.parse(
HttpRequest(
method=serialized_request.get("method") or "GET",
path=path,
query_string=to_str(query_string),
headers=headers,
body=body,
)
)
# Check if the determined operation_model is correct
assert parsed_operation_model == operation_model
# Check if the result is equal to the given "expected" dict or the kwargs (if "expected" has not been set)
expected = expected or kwargs
# The parser adds None for none-existing members on purpose. Remove those for the assert
expected = {key: value for key, value in expected.items() if value is not None}
parsed_request = {key: value for key, value in parsed_request.items() if value is not None}
assert parsed_request == expected
def test_query_parser():
"""Basic test for the QueryParser with a simple example (SQS SendMessage request)."""
parser = QueryRequestParser(load_service("sqs"))
request = HttpRequest(
body=to_bytes(
"Action=SendMessage&Version=2012-11-05&"
"QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&"
"MessageBody=%7B%22foo%22%3A+%22bared%22%7D&"
"DelaySeconds=2"),
method="POST",
headers={},
path="",
)
operation, params = parser.parse(request)
assert operation.name == "SendMessage"
assert params == {
"QueueUrl": "http://localhost:4566/000000000000/tf-acc-test-queue",
"MessageBody": '{"foo": "bared"}',
"DelaySeconds": 2,
}
def _set_request_props(request: HttpRequest, path: str, host: str):
"""Sets the HTTP request's path and host and clears the cache in the request object."""
request.path = path
request.headers["Host"] = host
try:
# delete the werkzeug request property cache that depends on path, but make sure all of them are
# initialized first, otherwise `del` will raise a key error
request.host = None # noqa
request.url = None # noqa
request.base_url = None # noqa
request.full_path = None # noqa
request.host_url = None # noqa
request.root_url = None # noqa
del request.host # noqa
del request.url # noqa
del request.base_url # noqa
del request.full_path # noqa
del request.host_url # noqa
del request.root_url # noqa
except AttributeError:
pass
def test_skeleton_e2e_sqs_send_message():
sqs_service = load_service("sqs")
skeleton = Skeleton(sqs_service, TestSqsApi())
context = RequestContext()
context.account = "test"
context.region = "us-west-1"
context.service = sqs_service
context.request = HttpRequest(
**{
"method": "POST",
"path": "/",
"body":
"Action=SendMessage&Version=2012-11-05&QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&MessageBody=%7B%22foo%22%3A+%22bared%22%7D&DelaySeconds=2",
"headers": _get_sqs_request_headers(),
})
result = skeleton.invoke(context)
# Use the parser from botocore to parse the serialized response
response_parser = create_parser(sqs_service.protocol)
parsed_response = response_parser.parse(
result.to_readonly_response_dict(),
sqs_service.operation_model("SendMessage").output_shape)
# Test the ResponseMetadata and delete it afterwards
assert "ResponseMetadata" in parsed_response
assert "RequestId" in parsed_response["ResponseMetadata"]
assert len(parsed_response["ResponseMetadata"]["RequestId"]) == 52
assert "HTTPStatusCode" in parsed_response["ResponseMetadata"]
assert parsed_response["ResponseMetadata"]["HTTPStatusCode"] == 200
del parsed_response["ResponseMetadata"]
# Compare the (remaining) actual payload
assert parsed_response == {
"MD5OfMessageBody": "String",
"MD5OfMessageAttributes": "String",
"MD5OfMessageSystemAttributes": "String",
"MessageId": "String",
"SequenceNumber": "String",
}
def _botocore_parser_integration_test(service: str,
action: str,
headers: dict = None,
expected: dict = None,
**kwargs):
# Load the appropriate service
service = load_service(service)
# Use the serializer from botocore to serialize the request params
serializer = create_serializer(service.protocol)
operation_model = service.operation_model(action)
serialized_request = serializer.serialize_to_request(
kwargs, operation_model)
body = serialized_request["body"]
query_string = urlencode(serialized_request.get("query_string") or "",
doseq=False)
# use custom headers (if provided), or headers from serialized request as default
headers = serialized_request.get("headers") if headers is None else headers
if service.protocol in ["query", "ec2"]:
# Serialize the body as query parameter
body = urlencode(serialized_request["body"])
# Use our parser to parse the serialized body
parser = create_parser(service)
parsed_operation_model, parsed_request = parser.parse(
HttpRequest(
method=serialized_request.get("method") or "GET",
path=serialized_request.get("url_path") or "",
query_string=query_string,
headers=headers,
body=body,
))
# Check if the determined operation_model is correct
assert parsed_operation_model == operation_model
# Check if the result is equal to the given "expected" dict or the kwargs (if "expected" has not been set)
assert parsed_request == (expected or kwargs)
def test_query_parser_non_flattened_map():
"""Simple test with a flattened map (SQS SetQueueAttributes request)."""
parser = QueryRequestParser(load_service("sns"))
request = HttpRequest(
body=to_bytes(
"Action=SetEndpointAttributes&"
"EndpointArn=arn%3Aaws%3Asns%3Aus-west-2%3A123456789012%3Aendpoint%2FGCM%2Fgcmpushapp%2F5e3e9847-3183-3f18-a7e8-671c3a57d4b3&"
"Attributes.entry.1.key=CustomUserData&"
"Attributes.entry.1.value=My+custom+userdata&"
"Version=2010-03-31&"
"AUTHPARAMS"
),
method="POST",
headers={},
path="",
)
operation, params = parser.parse(request)
assert operation.name == "SetEndpointAttributes"
assert params == {
"Attributes": {"CustomUserData": "My custom userdata"},
"EndpointArn": "arn:aws:sns:us-west-2:123456789012:endpoint/GCM/gcmpushapp/5e3e9847-3183-3f18-a7e8-671c3a57d4b3",
}
def test_skeleton_e2e_sqs_send_message_not_implemented():
sqs_service = load_service("sqs")
skeleton = Skeleton(sqs_service, TestSqsApiNotImplemented())
context = RequestContext()
context.account = "test"
context.region = "us-west-1"
context.service = sqs_service
context.request = HttpRequest(
**{
"method": "POST",
"path": "/",
"body":
"Action=SendMessage&Version=2012-11-05&QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&MessageBody=%7B%22foo%22%3A+%22bared%22%7D&DelaySeconds=2",
"headers": _get_sqs_request_headers(),
})
result = skeleton.invoke(context)
# Use the parser from botocore to parse the serialized response
response_parser = create_parser(sqs_service.protocol)
parsed_response = response_parser.parse(
result.to_readonly_response_dict(),
sqs_service.operation_model("SendMessage").output_shape)
# Test the ResponseMetadata
assert "ResponseMetadata" in parsed_response
assert "RequestId" in parsed_response["ResponseMetadata"]
assert len(parsed_response["ResponseMetadata"]["RequestId"]) == 52
assert "HTTPStatusCode" in parsed_response["ResponseMetadata"]
assert parsed_response["ResponseMetadata"]["HTTPStatusCode"] == 501
# Compare the (remaining) actual eror payload
assert "Error" in parsed_response
assert parsed_response["Error"] == {
"Code":
"InternalFailure",
"Message":
"API action 'SendMessage' for service 'sqs' not yet implemented",
}
def test_parser_error_on_unknown_error():
"""Test that the parser raises a UnknownParserError in case of an unknown exception."""
parser = QueryRequestParser(load_service("sqs"))
request = HttpRequest(
body=to_bytes(
"Action=SendMessage&Version=2012-11-05&"
"QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&"
"MessageBody=%7B%22foo%22%3A+%22bared%22%7D&"
"DelaySeconds=2"
),
method="POST",
headers={},
path="",
)
# An unknown error is obviously hard to trigger (because we would fix it if we would know of a way to trigger it),
# therefore we patch a function to raise an unexpected error
def raise_error(*args, **kwargs):
raise NotImplementedError()
parser._process_member = raise_error
with pytest.raises(UnknownParserError):
parser.parse(request)
def create_http_request(aws_request: AWSPreparedRequest) -> HttpRequest:
# create HttpRequest from AWSRequest
split_url = urlsplit(aws_request.url)
host = split_url.netloc.split(":")
if len(host) == 1:
server = (to_str(host[0]), None)
elif len(host) == 2:
server = (to_str(host[0]), int(host[1]))
else:
raise ValueError
# prepare the RequestContext
headers = Headers()
for k, v in aws_request.headers.items():
headers[k] = v
return HttpRequest(
method=aws_request.method,
path=split_url.path,
query_string=split_url.query,
headers=headers,
body=aws_request.body,
server=server,
)
def test_query_parser_flattened_map():
"""Simple test with a flattened map (SQS SetQueueAttributes request)."""
parser = QueryRequestParser(load_service("sqs"))
request = HttpRequest(
body=to_bytes(
"Action=SetQueueAttributes&Version=2012-11-05&"
"QueueUrl=http%3A%2F%2Flocalhost%3A4566%2F000000000000%2Ftf-acc-test-queue&"
"Attribute.1.Name=DelaySeconds&"
"Attribute.1.Value=10&"
"Attribute.2.Name=MaximumMessageSize&"
"Attribute.2.Value=131072&"
"Attribute.3.Name=MessageRetentionPeriod&"
"Attribute.3.Value=259200&"
"Attribute.4.Name=ReceiveMessageWaitTimeSeconds&"
"Attribute.4.Value=20&"
"Attribute.5.Name=RedrivePolicy&"
"Attribute.5.Value=%7B%22deadLetterTargetArn%22%3A%22arn%3Aaws%3Asqs%3Aus-east-1%3A80398EXAMPLE%3AMyDeadLetterQueue%22%2C%22maxReceiveCount%22%3A%221000%22%7D&"
"Attribute.6.Name=VisibilityTimeout&Attribute.6.Value=60"
),
method="POST",
headers={},
path="",
)
operation, params = parser.parse(request)
assert operation.name == "SetQueueAttributes"
assert params == {
"QueueUrl": "http://localhost:4566/000000000000/tf-acc-test-queue",
"Attributes": {
"DelaySeconds": "10",
"MaximumMessageSize": "131072",
"MessageRetentionPeriod": "259200",
"ReceiveMessageWaitTimeSeconds": "20",
"RedrivePolicy": '{"deadLetterTargetArn":"arn:aws:sqs:us-east-1:80398EXAMPLE:MyDeadLetterQueue","maxReceiveCount":"1000"}',
"VisibilityTimeout": "60",
},
}
