def is_s3_form_data(data_bytes):
if to_bytes("key=") in data_bytes:
return True
if (
to_bytes("Content-Disposition: form-data") in data_bytes
and to_bytes('name="key"') in data_bytes
):
return True
return False
def call_moto(context: RequestContext, include_response_metadata=False) -> ServiceResponse:
"""
Call moto with the given request context and receive a parsed ServiceResponse.
:param context: the request context
:param include_response_metadata: whether to include botocore's "ResponseMetadata" attribute
:return: a serialized AWS ServiceResponse (same as boto3 would return)
"""
status, headers, content = dispatch_to_moto(context)
operation_model = context.operation
response_dict = { # this is what botocore.endpoint.convert_to_response_dict normally does
"headers": dict(headers.items()), # boto doesn't like werkzeug headers
"status_code": status,
"body": to_bytes(content),
"context": {
"operation_name": operation_model.name,
},
}
parser = create_parser(context.service.protocol)
response = parser.parse(response_dict, operation_model.output_shape)
if status >= 301:
error = response["Error"]
raise CommonServiceException(
code=error.get("Code", "UnknownError"),
status_code=status,
message=error.get("Message", ""),
)
if not include_response_metadata:
response.pop("ResponseMetadata", None)
return response
def get_parameters_for_import(
self,
context: RequestContext,
key_id: KeyIdType,
wrapping_algorithm: AlgorithmSpec,
wrapping_key_spec: WrappingKeySpec,
) -> GetParametersForImportResponse:
key = _generate_data_key_pair({"KeySpec": wrapping_key_spec},
create_cipher=False,
add_to_keys=False)
import_token = short_uid()
import_state = KeyImportState(
key_id=key_id,
import_token=import_token,
private_key=key["PrivateKeyPlaintext"],
public_key=key["PublicKey"],
wrapping_algo=wrapping_algorithm,
key_obj=key["_key_"],
)
KMSBackend.get().imports[import_token] = import_state
expiry_date = datetime.datetime.now() + datetime.timedelta(days=100)
return GetParametersForImportResponse(
KeyId=key_id,
ImportToken=to_bytes(import_state.import_token),
PublicKey=import_state.public_key,
ParametersValidTo=expiry_date,
)
def _adjust_partition(self, source, static_partition: str = None):
# Call this function recursively if we get a dictionary or a list
if isinstance(source, dict):
result = {}
for k, v in source.items():
result[k] = self._adjust_partition(v, static_partition)
return result
if isinstance(source, list):
result = []
for v in source:
result.append(self._adjust_partition(v, static_partition))
return result
elif isinstance(source, bytes):
try:
decoded = unquote(to_str(source))
adjusted = self._adjust_partition(decoded, static_partition)
return to_bytes(adjusted)
except UnicodeDecodeError:
# If the body can't be decoded to a string, we return the initial source
return source
elif not isinstance(source, str):
# Ignore any other types
return source
return self.arn_regex.sub(
lambda m: self._adjust_match(m, static_partition), source)
def return_response(self, method, path, data, headers, response):
api = headers.get(HEADER_TARGET_API) or ""
if is_trace_logging_enabled(headers):
# print response trace for debugging, if enabled
if api and api != API_UNKNOWN:
LOG.debug(
'OUT(%s): "%s %s" - status: %s - response headers: %s - response: %s',
api,
method,
path,
response.status_code,
dict(response.headers),
response.content,
)
if (
response._content
and headers.get("Accept-Encoding") == "gzip"
and api not in SKIP_GZIP_APIS
and not response.headers.pop(HEADER_SKIP_RESPONSE_ZIPPING, None)
):
# services may decide to set HEADER_SKIP_RESPONSE_ZIPPING in the response, to skip result transformations
response._content = gzip.compress(to_bytes(response._content))
response.headers["Content-Length"] = str(len(response._content))
response.headers["Content-Encoding"] = "gzip"
def get_recordable_data(request_data):
if request_data or request_data in ["", b""]:
try:
request_data = to_bytes(request_data)
except Exception as ex:
LOG.warning("Unable to call to_bytes: %s", ex)
request_data = to_str(base64.b64encode(request_data))
return request_data
def _post_process_response_headers(response: RoutingResponse) -> None:
"""Adjust potential content lengths and checksums after modifying the response."""
if response.headers and response.content:
if "Content-Length" in response.headers:
response.headers["Content-Length"] = str(
len(to_bytes(response.content)))
if "x-amz-crc32" in response.headers:
response.headers["x-amz-crc32"] = calculate_crc32(
response.content)
def forward_request(self, method, path, data, headers):
# send request to target
url = f"{base_url}{path}"
response = requests.request(method=method,
url=url,
data=data,
headers=headers,
verify=False,
**requests_kwargs)
# fix encoding of response, based on Accept-Encoding header
if "gzip" in headers.get(HEADER_ACCEPT_ENCODING, "").lower():
response._content = gzip.compress(to_bytes(response._content))
response.headers["Content-Length"] = str(len(
response._content))
response.headers["Content-Encoding"] = "gzip"
return response
def convert_to_binary_event_payload(result,
event_type=None,
message_type=None):
# TODO This encoding has been migrated to the ASF Serializer.
# Remove this function after S3 and Kinesis have been migrated to ASF.
# e.g.: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
# e.g.: https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html
header_descriptors = {
":event-type": event_type or "Records",
":message-type": message_type or "event",
}
# construct headers
headers = b""
for key, value in header_descriptors.items():
header_name = key.encode(DEFAULT_ENCODING)
header_value = to_bytes(value)
headers += pack("!B", len(header_name))
headers += header_name
headers += pack("!B", AWS_BINARY_DATA_TYPE_STRING)
headers += pack("!H", len(header_value))
headers += header_value
# construct body
if isinstance(result, str):
body = bytes(result, DEFAULT_ENCODING)
else:
body = result
# calculate lengths
headers_length = len(headers)
body_length = len(body)
# construct message
result = pack("!I", body_length + headers_length + 16)
result += pack("!I", headers_length)
prelude_crc = binascii.crc32(result)
result += pack("!I", prelude_crc)
result += headers
result += body
payload_crc = binascii.crc32(result)
result += pack("!I", payload_crc)
return result
def dispatch_to_backend(
context: RequestContext,
http_request_dispatcher: Callable[[RequestContext], HttpBackendResponse],
include_response_metadata=False,
) -> ServiceResponse:
"""
Dispatch the given request to a backend by using the `request_forwarder` function to
fetch an HTTP response, converting it to a ServiceResponse.
:param context: the request context
:param http_request_dispatcher: dispatcher that performs the request and returns an HTTP response
:param include_response_metadata: whether to include boto3 response metadata in the response
:return:
"""
status, headers, content = http_request_dispatcher(context)
operation_model = context.operation
response_dict = { # this is what botocore.endpoint.convert_to_response_dict normally does
"headers": dict(headers.items()), # boto doesn't like werkzeug headers
"status_code": status,
"body": to_bytes(content),
"context": {
"operation_name": operation_model.name,
},
}
parser = create_parser(context.service.protocol)
response = parser.parse(response_dict, operation_model.output_shape)
if status >= 301:
error = response["Error"]
raise CommonServiceException(
code=error.get("Code", "UnknownError"),
status_code=status,
message=error.get("Message", ""),
sender_fault=("Type" in error),
)
if not include_response_metadata:
response.pop("ResponseMetadata", None)
return response
def convert_to_binary_event_payload(result,
event_type=None,
message_type=None):
# e.g.: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
# e.g.: https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html
header_descriptors = {
":event-type": event_type or "Records",
":message-type": message_type or "event",
}
# construct headers
headers = b""
for key, value in header_descriptors.items():
header_name = key.encode(DEFAULT_ENCODING)
header_value = to_bytes(value)
headers += pack("!B", len(header_name))
headers += header_name
headers += pack("!B", AWS_BINARY_DATA_TYPE_STRING)
headers += pack("!H", len(header_value))
headers += header_value
# construct body
body = bytes(result, DEFAULT_ENCODING)
# calculate lengths
headers_length = len(headers)
body_length = len(body)
# construct message
result = pack("!I", body_length + headers_length + 16)
result += pack("!I", headers_length)
prelude_crc = binascii.crc32(result)
result += pack("!I", prelude_crc)
result += headers
result += body
payload_crc = binascii.crc32(result)
result += pack("!I", payload_crc)
return result
def calculate_crc32(content):
return crc32(to_bytes(content)) & 0xFFFFFFFF
def send_event_to_target(
target_arn: str,
event: Dict,
target_attributes: Dict = None,
asynchronous: bool = True,
target: Dict = {},
):
region = target_arn.split(":")[3]
if ":lambda:" in target_arn:
from localstack.services.awslambda import lambda_api
lambda_api.run_lambda(func_arn=target_arn,
event=event,
context={},
asynchronous=asynchronous)
elif ":sns:" in target_arn:
sns_client = connect_to_service("sns", region_name=region)
sns_client.publish(TopicArn=target_arn, Message=json.dumps(event))
elif ":sqs:" in target_arn:
sqs_client = connect_to_service("sqs", region_name=region)
queue_url = get_sqs_queue_url(target_arn)
msg_group_id = dict_utils.get_safe(target_attributes,
"$.SqsParameters.MessageGroupId")
kwargs = {"MessageGroupId": msg_group_id} if msg_group_id else {}
sqs_client.send_message(QueueUrl=queue_url,
MessageBody=json.dumps(event),
**kwargs)
elif ":states:" in target_arn:
stepfunctions_client = connect_to_service("stepfunctions",
region_name=region)
stepfunctions_client.start_execution(stateMachineArn=target_arn,
input=json.dumps(event))
elif ":firehose:" in target_arn:
delivery_stream_name = firehose_name(target_arn)
firehose_client = connect_to_service("firehose", region_name=region)
firehose_client.put_record(
DeliveryStreamName=delivery_stream_name,
Record={"Data": to_bytes(json.dumps(event))},
)
elif ":events:" in target_arn:
if ":api-destination/" in target_arn or ":destination/" in target_arn:
send_event_to_api_destination(target_arn, event,
target.get("HttpParameters"))
else:
events_client = connect_to_service("events", region_name=region)
eventbus_name = target_arn.split(":")[-1].split("/")[-1]
events_client.put_events(
Entries=[{
"EventBusName": eventbus_name,
"Source": event.get("source"),
"DetailType": event.get("detail-type"),
"Detail": event.get("detail"),
}])
elif ":kinesis:" in target_arn:
partition_key_path = dict_utils.get_safe(
target_attributes,
"$.KinesisParameters.PartitionKeyPath",
default_value="$.id",
)
stream_name = target_arn.split("/")[-1]
partition_key = dict_utils.get_safe(event, partition_key_path,
event["id"])
kinesis_client = connect_to_service("kinesis", region_name=region)
kinesis_client.put_record(
StreamName=stream_name,
Data=to_bytes(json.dumps(event)),
PartitionKey=partition_key,
)
elif ":logs:" in target_arn:
log_group_name = target_arn.split(":")[-1]
logs_client = connect_to_service("logs", region_name=region)
log_stream_name = str(uuid.uuid4())
logs_client.create_log_stream(logGroupName=log_group_name,
logStreamName=log_stream_name)
logs_client.put_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
logEvents=[{
"timestamp": now_utc(millis=True),
"message": json.dumps(event)
}],
)
else:
LOG.warning('Unsupported Events rule target ARN: "%s"', target_arn)
def dummy_wsgi_environment(
method: str = "GET",
path: str = "",
headers: Optional[Union[Dict, Headers]] = None,
body: Optional[Union[bytes, str]] = None,
scheme: str = "http",
root_path: str = "/",
query_string: Optional[str] = None,
remote_addr: Optional[str] = None,
server: Optional[Tuple[str, Optional[int]]] = None,
raw_uri: Optional[str] = None,
) -> "WSGIEnvironment":
"""
Creates a dummy WSGIEnvironment that represents a standalone sans-IO HTTP requests.
See https://wsgi.readthedocs.io/en/latest/definitions.html#standard-environ-keys
:param method: The HTTP request method (such as GET or POST)
:param path: The remainder of the request URL's path. This may be an empty string, if the
request URL targets the application root and does not have a trailing slash.
:param headers: optional HTTP headers
:param body: the body of the request
:param scheme: the scheme (http or https)
:param root_path: The initial portion of the request URL's path that corresponds to the
application object.
:param query_string: The portion of the request URL that follows the “?”, if any. May be
empty or absent.
:param remote_addr: The address making the request
:param server: The server (tuple of server name and port)
:param raw_uri: The original path that may contain url encoded path elements.
:return: A WSGIEnvironment dictionary
"""
# Standard environ keys
environ = {
"REQUEST_METHOD": method,
# prepare the paths for the "WSGI decoding dance" done by werkzeug
"SCRIPT_NAME": unquote(quote(root_path.rstrip("/")), "latin-1"),
"PATH_INFO": unquote(quote(path), "latin-1"),
"SERVER_PROTOCOL": "HTTP/1.1",
}
data = strings.to_bytes(body) if body else b""
if query_string is not None:
environ["QUERY_STRING"] = query_string
if raw_uri:
if query_string:
raw_uri += "?" + query_string
environ["RAW_URI"] = raw_uri
environ["REQUEST_URI"] = environ["RAW_URI"]
if server:
environ["SERVER_NAME"] = server[0]
if server[1]:
environ["SERVER_PORT"] = str(server[1])
else:
environ["SERVER_PORT"] = "80"
else:
environ["SERVER_NAME"] = "127.0.0.1"
environ["SERVER_PORT"] = "80"
if remote_addr:
environ["REMOTE_ADDR"] = remote_addr
if headers:
for k, v in headers.items():
name = k.upper().replace("-", "_")
if name not in ("CONTENT_TYPE", "CONTENT_LENGTH"):
name = f"HTTP_{name}"
val = v
if name in environ:
val = environ[name] + "," + val
environ[name] = val
if "CONTENT_LENGTH" not in environ:
# try to determine content length from body
environ["CONTENT_LENGTH"] = str(len(data))
# WSGI environ keys
environ["wsgi.version"] = (1, 0)
environ["wsgi.url_scheme"] = scheme
environ["wsgi.input"] = BytesIO(data)
environ["wsgi.input_terminated"] = True
environ["wsgi.errors"] = BytesIO()
environ["wsgi.multithread"] = True
environ["wsgi.multiprocess"] = False
environ["wsgi.run_once"] = False
return environ
def invoke(fn, self, *args, **kwargs):
payload = to_bytes(args[0])
client = aws_stack.connect_to_service("lambda")
return client.invoke(FunctionName=self.function_name, Payload=payload)
def invoke(self, invocation_context: ApiInvocationContext):
uri = (invocation_context.integration.get("uri")
or invocation_context.integration.get("integrationUri") or "")
relative_path, query_string_params = extract_query_string_params(
path=invocation_context.path_with_query_string)
api_id = invocation_context.api_id
stage = invocation_context.stage
headers = invocation_context.headers
resource_path = invocation_context.resource_path
invocation_context.context = get_event_request_context(
invocation_context)
try:
path_params = extract_path_params(path=relative_path,
extracted_path=resource_path)
invocation_context.path_params = path_params
except Exception:
path_params = {}
func_arn = uri
if ":lambda:path" in uri:
func_arn = uri.split(":lambda:path")[1].split(
"functions/")[1].split("/invocations")[0]
if invocation_context.authorizer_type:
authorizer_context = {
invocation_context.authorizer_type:
invocation_context.auth_context
}
invocation_context.context["authorizer"] = authorizer_context
payload = self.request_templates.render(invocation_context)
# TODO: change this signature to InvocationContext as well!
result = lambda_api.process_apigateway_invocation(
func_arn,
relative_path,
payload,
stage,
api_id,
headers,
is_base64_encoded=invocation_context.is_data_base64_encoded,
path_params=path_params,
query_string_params=query_string_params,
method=invocation_context.method,
resource_path=resource_path,
request_context=invocation_context.context,
stage_variables=invocation_context.stage_variables,
)
if isinstance(result, FlaskResponse):
response = flask_to_requests_response(result)
elif isinstance(result, Response):
response = result
else:
response = LambdaResponse()
parsed_result = result if isinstance(result, dict) else json.loads(
str(result or "{}"))
parsed_result = common.json_safe(parsed_result)
parsed_result = {} if parsed_result is None else parsed_result
response.status_code = int(parsed_result.get("statusCode", 200))
parsed_headers = parsed_result.get("headers", {})
if parsed_headers is not None:
response.headers.update(parsed_headers)
try:
result_body = parsed_result.get("body")
if isinstance(result_body, dict):
response._content = json.dumps(result_body)
else:
body_bytes = to_bytes(to_str(result_body or ""))
if parsed_result.get("isBase64Encoded", False):
body_bytes = base64.b64decode(body_bytes)
response._content = body_bytes
except Exception as e:
LOG.warning("Couldn't set Lambda response content: %s", e)
response._content = "{}"
response.multi_value_headers = parsed_result.get(
"multiValueHeaders") or {}
# apply custom response template
self.update_content_length(response)
invocation_context.response = response
self.response_templates.render(invocation_context)
return invocation_context.response
def start_container(
self,
container_name_or_id: str,
stdin=None,
interactive: bool = False,
attach: bool = False,
flags: Optional[str] = None,
) -> Tuple[bytes, bytes]:
LOG.debug("Starting container %s", container_name_or_id)
try:
container = self.client().containers.get(container_name_or_id)
stdout = to_bytes(container_name_or_id)
stderr = b""
if interactive or attach:
params = {"stdout": 1, "stderr": 1, "stream": 1}
if interactive:
params["stdin"] = 1
sock = container.attach_socket(params=params)
sock = sock._sock if hasattr(sock, "_sock") else sock
result_queue = queue.Queue()
thread_started = threading.Event()
start_waiting = threading.Event()
# Note: We need to be careful about potential race conditions here - .wait() should happen right
# after .start(). Hence starting a thread and asynchronously waiting for the container exit code
def wait_for_result(*_):
_exit_code = -1
try:
thread_started.set()
start_waiting.wait()
_exit_code = container.wait()["StatusCode"]
except APIError as e:
_exit_code = 1
raise ContainerException(str(e))
finally:
result_queue.put(_exit_code)
# start listener thread
start_worker_thread(wait_for_result)
thread_started.wait()
# start container
container.start()
# start awaiting container result
start_waiting.set()
# handle container input/output
# under windows, the socket has no __enter__ / cannot be used as context manager
# therefore try/finally instead of with here
try:
if stdin:
sock.sendall(to_bytes(stdin))
sock.shutdown(socket.SHUT_WR)
stdout, stderr = self._read_from_sock(sock, False)
except socket.timeout:
LOG.debug(
f"Socket timeout when talking to the I/O streams of Docker container '{container_name_or_id}'"
)
finally:
sock.close()
# get container exit code
exit_code = result_queue.get()
if exit_code:
raise ContainerException(
f"Docker container returned with exit code {exit_code}",
stdout=stdout,
stderr=stderr,
)
else:
container.start()
return stdout, stderr
except NotFound:
raise NoSuchContainer(container_name_or_id)
except APIError as e:
raise ContainerException() from e
def roundtrip(data):
encoded = base64.urlsafe_b64encode(to_bytes(data))
result = base64_decode(encoded)
assert to_bytes(data) == result
def _in_path_or_payload(search_str):
return to_str(search_str) in path or to_bytes(search_str) in data_bytes
def get_api_from_custom_rules(method, path, data, headers):
"""Determine backend port based on custom rules."""
# API Gateway invocation URLs
if ("/%s/" % PATH_USER_REQUEST) in path:
return "apigateway", config.service_port("apigateway")
# detect S3 presigned URLs
if "AWSAccessKeyId=" in path or "Signature=" in path:
return "s3", config.service_port("s3")
# heuristic for SQS queue URLs
if is_sqs_queue_url(path):
return "sqs", config.service_port("sqs")
# DynamoDB shell URLs
if path.startswith("/shell") or path.startswith("/dynamodb/shell"):
return "dynamodb", config.service_port("dynamodb")
data_bytes = to_bytes(data or "")
version, action = extract_version_and_action(path, data_bytes)
def _in_path_or_payload(search_str):
return to_str(search_str) in path or to_bytes(search_str) in data_bytes
if path == "/" and b"QueueName=" in data_bytes:
return "sqs", config.service_port("sqs")
if "Action=ConfirmSubscription" in path:
return "sns", config.service_port("sns")
if path.startswith("/2015-03-31/functions/"):
return "lambda", config.service_port("lambda")
if _in_path_or_payload("Action=AssumeRoleWithWebIdentity"):
return "sts", config.service_port("sts")
if _in_path_or_payload("Action=AssumeRoleWithSAML"):
return "sts", config.service_port("sts")
if _in_path_or_payload("Action=AssumeRole"):
return "sts", config.service_port("sts")
# SQS queue requests
if _in_path_or_payload("QueueUrl=") and _in_path_or_payload("Action="):
return "sqs", config.service_port("sqs")
if matches_service_action("sqs", action, version=version):
return "sqs", config.service_port("sqs")
# SNS topic requests
if matches_service_action("sns", action, version=version):
return "sns", config.service_port("sns")
# TODO: move S3 public URLs to a separate port/endpoint, OR check ACLs here first
stripped = path.strip("/")
if method in ["GET", "HEAD"] and stripped:
# assume that this is an S3 GET request with URL path `/<bucket>/<key ...>`
return "s3", config.service_port("s3")
# detect S3 URLs
if stripped and "/" not in stripped:
if method == "PUT":
# assume that this is an S3 PUT bucket request with URL path `/<bucket>`
return "s3", config.service_port("s3")
if method == "POST" and is_s3_form_data(data_bytes):
# assume that this is an S3 POST request with form parameters or multipart form in the body
return "s3", config.service_port("s3")
# detect S3 requests sent from aws-cli using --no-sign-request option
if "aws-cli/" in headers.get("User-Agent", ""):
return "s3", config.service_port("s3")
# S3 delete object requests
if (
method == "POST"
and "delete=" in path
and b"<Delete" in data_bytes
and b"<Key>" in data_bytes
):
return "s3", config.service_port("s3")
# Put Object API can have multiple keys
if stripped.count("/") >= 1 and method == "PUT":
# assume that this is an S3 PUT bucket object request with URL path `/<bucket>/object`
# or `/<bucket>/object/object1/+`
return "s3", config.service_port("s3")
auth_header = headers.get("Authorization") or ""
# detect S3 requests with "AWS id:key" Auth headers
if auth_header.startswith("AWS "):
return "s3", config.service_port("s3")
# certain EC2 requests from Java SDK contain no Auth headers (issue #3805)
if b"Version=2016-11-15" in data_bytes:
return "ec2", config.service_port("ec2")
def get_hash(value) -> str:
max_length = 10
digest = hashlib.sha1()
digest.update(to_bytes(str(value)))
result = digest.hexdigest()
return result[:max_length]
def legacy_rules(request: Request) -> Optional[str]:
"""
*Legacy* rules which migrate routing logic which will become obsolete with the ASF Gateway.
All rules which are implemented here should be migrated to the new router once these services are migrated to ASF.
TODO: These custom rules should become obsolete by migrating these to use the http/router.py
"""
path = request.path
method = request.method
host = hostname_from_url(request.host)
# API Gateway invocation URLs
# TODO: deprecated with #6040, where API GW user routes are served through the gateway directly
if ("/%s/" % PATH_USER_REQUEST) in request.path or (
host.endswith(LOCALHOST_HOSTNAME) and "execute-api" in host):
return "apigateway"
# DynamoDB shell URLs
if path.startswith("/shell") or path.startswith("/dynamodb/shell"):
return "dynamodb"
# TODO The remaining rules here are special S3 rules - needs to be discussed how these should be handled.
# Some are similar to other rules and not that greedy, others are nearly general fallbacks.
stripped = path.strip("/")
if method in ["GET", "HEAD"] and stripped:
# assume that this is an S3 GET request with URL path `/<bucket>/<key ...>`
return "s3"
# detect S3 URLs
if stripped and "/" not in stripped:
if method == "PUT":
# assume that this is an S3 PUT bucket request with URL path `/<bucket>`
return "s3"
if method == "POST" and "key" in request.values:
# assume that this is an S3 POST request with form parameters or multipart form in the body
return "s3"
# detect S3 requests sent from aws-cli using --no-sign-request option
if "aws-cli/" in str(request.user_agent):
return "s3"
# detect S3 pre-signed URLs (v2 and v4)
values = request.values
if any(value in values for value in [
"AWSAccessKeyId",
"Signature",
"X-Amz-Algorithm",
"X-Amz-Credential",
"X-Amz-Date",
"X-Amz-Expires",
"X-Amz-SignedHeaders",
"X-Amz-Signature",
]):
return "s3"
# S3 delete object requests
if method == "POST" and "delete" in values:
data_bytes = to_bytes(request.data)
if b"<Delete" in data_bytes and b"<Key>" in data_bytes:
return "s3"
# Put Object API can have multiple keys
if stripped.count("/") >= 1 and method == "PUT":
# assume that this is an S3 PUT bucket object request with URL path `/<bucket>/object`
# or `/<bucket>/object/object1/+`
return "s3"
# detect S3 requests with "AWS id:key" Auth headers
auth_header = request.headers.get("Authorization") or ""
if auth_header.startswith("AWS "):
return "s3"
if uses_host_addressing(request.headers):
# Note: This needs to be the last rule (and therefore is not in the host rules), since it is incredibly greedy
return "s3"
# maps service names/versions to list of action names
SERVICE_ACTIONS_CACHE: Dict[str, Set[str]] = {}
# default service versions
DEFAULT_SERVICE_VERSIONS: Dict[str, str] = {
"sns": "2010-03-31",
"sqs": "2012-11-05",
}
# regexes to extract info from URL paths / payloads
_REGEX_ACTION = r"(^|.*\?|.*&)Action=([a-zA-Z0-9_]+)($|&)"
_REGEX_VERSION = r"(^|.*\?|.*&)Version=([a-zA-Z0-9_]+)($|&)"
REGEX_ACTION: re.Pattern = re.compile(_REGEX_ACTION)
REGEX_VERSION: re.Pattern = re.compile(_REGEX_VERSION)
REGEXB_ACTION: re.Pattern = re.compile(to_bytes(_REGEX_ACTION))
REGEXB_VERSION: re.Pattern = re.compile(to_bytes(_REGEX_VERSION))
# TODO: Add more comprehensive tests for AWS SDK v2. It seems that
# the v2 SDK (e.g., from Java) in certain configurations is not sending
# the Authorization header we depend on. We can use some of the heuristics
# below for routing requests to the correct target services (based on
# 'Action' or 'Version' attributes in the request), but the more severe issue
# seems to be that the region info is not being transmitted in certain
# situations. If this turns out to be true, then we may need to think about
# a more comprehensive refactoring of our routing/region-targeting approach.
def get_service_action_names(service: str, version: str = None) -> Set[str]:
"""Returns, for a given service name and version, the list of available service action names."""
version = version or DEFAULT_SERVICE_VERSIONS.get(service)
def test_kinesis_event_source_mapping_with_on_failure_destination_config(
self,
lambda_client,
create_lambda_function,
sqs_client,
sqs_queue_arn,
sqs_create_queue,
create_iam_role_with_policy,
kinesis_client,
wait_for_stream_ready,
):
try:
function_name = f"lambda_func-{short_uid()}"
role = f"test-lambda-role-{short_uid()}"
policy_name = f"test-lambda-policy-{short_uid()}"
kinesis_name = f"test-kinesis-{short_uid()}"
role_arn = create_iam_role_with_policy(
RoleName=role,
PolicyName=policy_name,
RoleDefinition=lambda_role,
PolicyDefinition=s3_lambda_permission,
)
create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON37,
role=role_arn,
)
kinesis_client.create_stream(StreamName=kinesis_name, ShardCount=1)
result = kinesis_client.describe_stream(
StreamName=kinesis_name)["StreamDescription"]
kinesis_arn = result["StreamARN"]
wait_for_stream_ready(stream_name=kinesis_name)
queue_event_source_mapping = sqs_create_queue()
destination_queue = sqs_queue_arn(queue_event_source_mapping)
destination_config = {
"OnFailure": {
"Destination": destination_queue
}
}
message = {
"input": "hello",
"value": "world",
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG: 1,
}
result = lambda_client.create_event_source_mapping(
FunctionName=function_name,
BatchSize=1,
StartingPosition="LATEST",
EventSourceArn=kinesis_arn,
MaximumBatchingWindowInSeconds=1,
MaximumRetryAttempts=1,
DestinationConfig=destination_config,
)
event_source_mapping_uuid = result["UUID"]
_await_event_source_mapping_enabled(lambda_client,
event_source_mapping_uuid)
kinesis_client.put_record(StreamName=kinesis_name,
Data=to_bytes(json.dumps(message)),
PartitionKey="custom")
def verify_failure_received():
result = sqs_client.receive_message(
QueueUrl=queue_event_source_mapping)
msg = result["Messages"][0]
body = json.loads(msg["Body"])
assert body["requestContext"][
"condition"] == "RetryAttemptsExhausted"
assert body["KinesisBatchInfo"]["batchSize"] == 1
assert body["KinesisBatchInfo"]["streamArn"] == kinesis_arn
retry(verify_failure_received, retries=50, sleep=5, sleep_before=5)
finally:
kinesis_client.delete_stream(StreamName=kinesis_name,
EnforceConsumerDeletion=True)
lambda_client.delete_event_source_mapping(
UUID=event_source_mapping_uuid)
def calculate_crc32(response):
return crc32(to_bytes(response.content)) & 0xFFFFFFFF
