def test_invoke_apis_via_edge(self):
edge_url = '%s://localhost:%s' % (get_service_protocol(),
config.EDGE_PORT)
if is_api_enabled('s3'):
self._invoke_s3_via_edge(edge_url)
if is_api_enabled('kinesis'):
self._invoke_kinesis_via_edge(edge_url)
if is_api_enabled('dynamodbstreams'):
self._invoke_dynamodbstreams_via_edge(edge_url)
if is_api_enabled('firehose'):
self._invoke_firehose_via_edge(edge_url)
if is_api_enabled('stepfunctions'):
self._invoke_stepfunctions_via_edge(edge_url)
def test_invoke_apis_via_edge(self):
edge_port = config.EDGE_PORT_HTTP or config.EDGE_PORT
edge_url = "%s://localhost:%s" % (get_service_protocol(), edge_port)
if is_api_enabled("s3"):
self._invoke_s3_via_edge(edge_url)
self._invoke_s3_via_edge_multipart_form(edge_url)
if is_api_enabled("kinesis"):
self._invoke_kinesis_via_edge(edge_url)
if is_api_enabled("dynamodbstreams"):
self._invoke_dynamodbstreams_via_edge(edge_url)
if is_api_enabled("firehose"):
self._invoke_firehose_via_edge(edge_url)
if is_api_enabled("stepfunctions"):
self._invoke_stepfunctions_via_edge(edge_url)
def store_cloudwatch_logs(
log_group_name,
log_stream_name,
log_output,
start_time=None,
auto_create_group: Optional[bool] = True,
):
if not is_api_enabled("logs"):
return
start_time = start_time or int(time.time() * 1000)
logs_client = aws_stack.connect_to_service("logs")
log_output = to_str(log_output)
if auto_create_group:
# make sure that the log group exists, create it if not
log_groups = logs_client.describe_log_groups()["logGroups"]
log_groups = [lg["logGroupName"] for lg in log_groups]
if log_group_name not in log_groups:
try:
logs_client.create_log_group(logGroupName=log_group_name)
except Exception as e:
if "ResourceAlreadyExistsException" in str(e):
# this can happen in certain cases, possibly due to a race condition
pass
else:
raise e
# create a new log stream for this lambda invocation
try:
logs_client.create_log_stream(logGroupName=log_group_name,
logStreamName=log_stream_name)
except Exception: # TODO: narrow down
pass
# store new log events under the log stream
finish_time = int(time.time() * 1000)
# fix for log lines that were merged into a singe line, e.g., "log line 1 ... \x1b[32mEND RequestId ..."
log_output = log_output.replace("\\x1b", "\n\\x1b")
log_output = log_output.replace("\x1b", "\n\x1b")
log_lines = log_output.split("\n")
time_diff_per_line = float(finish_time - start_time) / float(
len(log_lines))
log_events = []
for i, line in enumerate(log_lines):
if not line:
continue
# simple heuristic: assume log lines were emitted in regular intervals
log_time = start_time + float(i) * time_diff_per_line
event = {"timestamp": int(log_time), "message": line}
log_events.append(event)
if not log_events:
return
logs_client.put_log_events(logGroupName=log_group_name,
logStreamName=log_stream_name,
logEvents=log_events)
def delete_all_event_source_mappings(self, table_arn):
if table_arn:
# fix start dynamodb service without lambda
if not is_api_enabled('lambda'):
return
lambda_client = aws_stack.connect_to_service('lambda')
result = lambda_client.list_event_source_mappings(EventSourceArn=table_arn)
for event in result['EventSourceMappings']:
event_source_mapping_id = event['UUID']
lambda_client.delete_event_source_mapping(UUID=event_source_mapping_id)
def _notify_event_subscribers(name: ParameterName, operation: str):
"""Publish an EventBridge event to notify subscribers of changes."""
if not is_api_enabled("events"):
return
events = aws_stack.connect_to_service("events")
detail = {"name": name, "operation": operation}
event = {
"Source": "aws.ssm",
"Detail": json.dumps(detail),
"DetailType": "Parameter Store Change",
}
events.put_events(Entries=[event])
def replay_command(command):
api = command['a']
if not is_api_enabled(api):
return
function = getattr(requests, command['m'].lower())
data = prepare_replay_data(command)
endpoint = aws_stack.get_local_service_url(api)
full_url = (endpoint[:-1]
if endpoint.endswith('/') else endpoint) + command['p']
response = function(full_url,
data=data,
headers=command['h'],
verify=False)
return response
def replay_command(command):
api = command['a']
if not is_api_enabled(api):
return
function = getattr(requests, command['m'].lower())
data = prepare_replay_data(command)
endpoint = aws_stack.get_local_service_url(api)
full_url = (endpoint[:-1] if endpoint.endswith('/') else endpoint) + command['p']
try:
# fix an error when calling requests with invalid payload encoding
data and hasattr(data, 'encode') and data.encode('latin-1')
except UnicodeEncodeError:
if hasattr(data, 'encode'):
data = data.encode('utf-8')
response = function(full_url, data=data, headers=command['h'], verify=False)
return response
def replay_command(command):
api = command["a"]
if not is_api_enabled(api):
return
function = getattr(requests, command["m"].lower())
data = prepare_replay_data(command)
endpoint = aws_stack.get_local_service_url(api)
full_url = (endpoint[:-1] if endpoint.endswith("/") else endpoint) + command["p"]
headers = aws_stack.set_internal_auth(command["h"])
try:
# fix an error when calling requests with invalid payload encoding
data and hasattr(data, "encode") and data.encode("latin-1")
except UnicodeEncodeError:
if hasattr(data, "encode"):
data = data.encode("utf-8")
response = function(full_url, data=data, headers=headers, verify=False)
return response
def notify_event_subscribers(data, target_header):
"""Publish an EventBridge event to notify subscribers of changes."""
if not is_api_enabled("events"):
return
events = aws_stack.connect_to_service("events")
operation = EVENT_BRIDGE_OPERATIONS.get(target_header)
if not operation:
LOG.warning(
"Unexpected target header '%s' when sending EventBridge notification from SSM",
target_header,
)
detail = {"name": data["Name"], "operation": operation}
event = {
"Source": "aws.ssm",
"Detail": json.dumps(detail),
"DetailType": "Parameter Store Change",
}
events.put_events(Entries=[event])
def test_invoke_apis_via_edge(self):
edge_url = config.get_edge_url()
if is_api_enabled("s3"):
self._invoke_s3_via_edge(edge_url)
self._invoke_s3_via_edge_multipart_form(edge_url)
if is_api_enabled("kinesis"):
self._invoke_kinesis_via_edge(edge_url)
if is_api_enabled("dynamodbstreams"):
self._invoke_dynamodb_via_edge_go_sdk(edge_url)
if is_api_enabled("dynamodbstreams"):
self._invoke_dynamodbstreams_via_edge(edge_url)
if is_api_enabled("firehose"):
self._invoke_firehose_via_edge(edge_url)
if is_api_enabled("stepfunctions"):
self._invoke_stepfunctions_via_edge(edge_url)
def is_enabled(self):
if self.default_active:
return True
return is_api_enabled(self.name())
class TestEdgeAPI:
@pytest.mark.skipif(not is_api_enabled("kinesis"), reason="kinesis not enabled")
def test_invoke_kinesis(self):
edge_url = config.get_edge_url()
self._invoke_kinesis_via_edge(edge_url)
@pytest.mark.skipif(not is_api_enabled("dynamodbstreams"), reason="dynamodbstreams not enabled")
def test_invoke_dynamodb(self):
edge_url = config.get_edge_url()
self._invoke_dynamodb_via_edge_go_sdk(edge_url)
@pytest.mark.skipif(not is_api_enabled("dynamodbstreams"), reason="dynamodbstreams not enabled")
def test_invoke_dynamodbstreams(self):
edge_url = config.get_edge_url()
self._invoke_dynamodbstreams_via_edge(edge_url)
@pytest.mark.skipif(not is_api_enabled("firehose"), reason="firehose not enabled")
def test_invoke_firehose(self):
edge_url = config.get_edge_url()
self._invoke_firehose_via_edge(edge_url)
@pytest.mark.skipif(not is_api_enabled("stepfunctions"), reason="stepfunctions not enabled")
def test_invoke_stepfunctions(self):
edge_url = config.get_edge_url()
self._invoke_stepfunctions_via_edge(edge_url)
@pytest.mark.skipif(not is_api_enabled("s3"), reason="s3 not enabled")
def test_invoke_s3(self):
edge_url = config.get_edge_url()
self._invoke_s3_via_edge(edge_url)
@pytest.mark.skipif(not is_api_enabled("s3"), reason="s3 not enabled")
@pytest.mark.xfail(
condition=not config.LEGACY_EDGE_PROXY, reason="failing with new HTTP gateway (only in CI)"
)
def test_invoke_s3_multipart_request(self):
edge_url = config.get_edge_url()
self._invoke_s3_via_edge_multipart_form(edge_url)
def _invoke_kinesis_via_edge(self, edge_url):
client = aws_stack.create_external_boto_client("kinesis", endpoint_url=edge_url)
result = client.list_streams()
assert "StreamNames" in result
def _invoke_dynamodbstreams_via_edge(self, edge_url):
client = aws_stack.create_external_boto_client("dynamodbstreams", endpoint_url=edge_url)
result = client.list_streams()
assert "Streams" in result
def _invoke_firehose_via_edge(self, edge_url):
client = aws_stack.create_external_boto_client("firehose", endpoint_url=edge_url)
result = client.list_delivery_streams()
assert "DeliveryStreamNames" in result
def _invoke_stepfunctions_via_edge(self, edge_url):
client = aws_stack.create_external_boto_client("stepfunctions", endpoint_url=edge_url)
result = client.list_state_machines()
assert "stateMachines" in result
def _invoke_dynamodb_via_edge_go_sdk(self, edge_url):
client = aws_stack.create_external_boto_client("dynamodb")
table_name = f"t-{short_uid()}"
aws_stack.create_dynamodb_table(table_name, "id")
# emulate a request sent from the AWS Go SDK v2
headers = {
"Host": "awsmock:4566",
"User-Agent": "aws-sdk-go-v2/1.9.0 os/linux lang/go/1.16.7 md/GOOS/linux md/GOARCH/amd64 api/dynamodb/1.5.0",
"Accept-Encoding": "identity",
"Amz-Sdk-Invocation-Id": "af832536-dbc7-436e-9d6d-60840a0ff203",
"Amz-Sdk-Request": "attempt=1; max=3",
"Content-Type": "application/x-amz-json-1.0",
"X-Amz-Target": "DynamoDB_20120810.DescribeTable",
}
data = json.dumps({"TableName": table_name})
response = requests.post(edge_url, data=data, headers=headers)
assert response.status_code == 200
content = json.loads(to_str(response.content))
assert content.get("Table")
# clean up
client.delete_table(TableName=table_name)
def _invoke_s3_via_edge(self, edge_url):
client = aws_stack.create_external_boto_client("s3", endpoint_url=edge_url)
bucket_name = "edge-%s" % short_uid()
client.create_bucket(Bucket=bucket_name)
result = client.head_bucket(Bucket=bucket_name)
assert result["ResponseMetadata"]["HTTPStatusCode"] == 200
client.delete_bucket(Bucket=bucket_name)
bucket_name = "edge-%s" % short_uid()
object_name = "testobject"
bucket_url = "%s/%s" % (edge_url, bucket_name)
result = requests.put(bucket_url, verify=False)
assert result.status_code == 200
result = client.head_bucket(Bucket=bucket_name)
assert result["ResponseMetadata"]["HTTPStatusCode"] == 200
headers = {"Content-Type": "application/x-www-form-urlencoded"}
result = requests.post(
bucket_url,
data="key=%s&file=file_content_123" % object_name,
headers=headers,
verify=False,
)
assert result.status_code == 204
bucket_url = "%s/example" % bucket_url
result = requests.put(bucket_url, data="hello", verify=False)
assert result.status_code == 200
result = io.BytesIO()
client.download_fileobj(bucket_name, object_name, result)
assert to_str(result.getvalue()) == "file_content_123"
def _invoke_s3_via_edge_multipart_form(self, edge_url):
client = aws_stack.create_external_boto_client("s3", endpoint_url=edge_url)
bucket_name = "edge-%s" % short_uid()
object_name = "testobject"
object_data = b"testdata"
client.create_bucket(Bucket=bucket_name)
presigned_post = client.generate_presigned_post(bucket_name, object_name)
files = {"file": object_data}
r = requests.post(
presigned_post["url"],
data=presigned_post["fields"],
files=files,
verify=False,
)
assert r.status_code == 204
result = io.BytesIO()
client.download_fileobj(bucket_name, object_name, result)
assert to_str(result.getvalue()) == to_str(object_data)
client.delete_object(Bucket=bucket_name, Key=object_name)
client.delete_bucket(Bucket=bucket_name)
def test_basic_https_invocation(self):
class MyListener(ProxyListener):
def forward_request(self, method, path, data, headers):
return {"method": method, "path": path, "data": data}
port = get_free_tcp_port()
url = f"https://localhost:{port}/foo/bar"
listener = MyListener()
proxy = start_proxy_server(port, update_listener=listener, use_ssl=True)
response = requests.post(url, verify=False)
expected = {"method": "POST", "path": "/foo/bar", "data": ""}
assert json.loads(to_str(response.content)) == expected
proxy.stop()
def test_http2_relay_traffic(self):
"""Tests if HTTP2 traffic can correctly be forwarded (including url-encoded characters)."""
# Create a simple HTTP echo server
class MyListener(ProxyListener):
def forward_request(self, method, path, data, headers):
return {"method": method, "path": path, "data": data}
listener = MyListener()
port_http_server = get_free_tcp_port()
http_server = start_proxy_server(port_http_server, update_listener=listener, use_ssl=True)
# Create a relay proxy which forwards request to the HTTP echo server
port_relay_proxy = get_free_tcp_port()
forward_url = f"https://localhost:{port_http_server}"
relay_proxy = start_proxy_server(port_relay_proxy, forward_url=forward_url, use_ssl=True)
# Contact the relay proxy
query = "%2B=%3B%2C%2F%3F%3A%40%26%3D%2B%24%21%2A%27%28%29%23"
path = f"/foo/bar%3B%2C%2F%3F%3A%40%26%3D%2B%24%21%2A%27%28%29%23baz?{query}"
url = f"https://localhost:{port_relay_proxy}{path}"
response = requests.post(url, verify=False)
# Expect the response from the HTTP echo server
expected = {
"method": "POST",
"path": path,
"data": "",
}
assert json.loads(to_str(response.content)) == expected
http_server.stop()
relay_proxy.stop()
def test_invoke_sns_sqs_integration_using_edge_port(
self, sqs_create_queue, sqs_client, sns_client, sns_create_topic, sns_subscription
):
topic_name = f"topic-{short_uid()}"
queue_name = f"queue-{short_uid()}"
region_original = os.environ.get("DEFAULT_REGION")
os.environ["DEFAULT_REGION"] = "us-southeast-2"
topic = sns_client.create_topic(Name=topic_name)
topic_arn = topic["TopicArn"]
queue_url = sqs_create_queue(QueueName=queue_name)
sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=["QueueArn"])
sns_subscription(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_url)
sns_client.publish(TargetArn=topic_arn, Message="Test msg")
response = sqs_client.receive_message(
QueueUrl=queue_url,
AttributeNames=["SentTimestamp"],
MaxNumberOfMessages=1,
MessageAttributeNames=["All"],
VisibilityTimeout=2,
WaitTimeSeconds=2,
)
assert len(response["Messages"]) == 1
os.environ.pop("DEFAULT_REGION")
if region_original is not None:
os.environ["DEFAULT_REGION"] = region_original
def test_message_modifying_handler(self, s3_client, monkeypatch):
class MessageModifier(MessageModifyingProxyListener):
def forward_request(self, method, path: str, data, headers):
if method != "HEAD":
return Request(path=path.replace(bucket_name, f"{bucket_name}-patched"))
def return_response(self, method, path, data, headers, response):
if method == "HEAD":
return Response(status_code=201)
content = to_str(response.content or "")
if "test content" in content:
return Response(content=content + " patched")
updated_handlers = list(ProxyListener.DEFAULT_LISTENERS) + [MessageModifier()]
monkeypatch.setattr(ProxyListener, "DEFAULT_LISTENERS", updated_handlers)
# create S3 bucket, assert that patched bucket name is used
bucket_name = f"b-{short_uid()}"
s3_client.create_bucket(Bucket=bucket_name)
buckets = [b["Name"] for b in s3_client.list_buckets()["Buckets"]]
assert f"{bucket_name}-patched" in buckets
assert f"{bucket_name}" not in buckets
result = s3_client.head_bucket(Bucket=f"{bucket_name}-patched")
assert result["ResponseMetadata"]["HTTPStatusCode"] == 201
# put content, assert that patched content is returned
key = "test/1/2/3"
s3_client.put_object(Bucket=bucket_name, Key=key, Body=b"test content 123")
result = s3_client.get_object(Bucket=bucket_name, Key=key)
content = to_str(result["Body"].read())
assert " patched" in content
def test_handler_returning_none_method(self, s3_client, monkeypatch):
class MessageModifier(ProxyListener):
def forward_request(self, method, path: str, data, headers):
# simple heuristic to determine whether we are in the context of an edge call, or service request
is_edge_request = not headers.get(HEADER_LOCALSTACK_EDGE_URL)
if not is_edge_request and method == "PUT" and len(path.split("/")) > 3:
# simple test that asserts we can forward a Request object with only URL and empty/None method
return RequestsRequest(method=None, data=to_str(data) + " patched")
return True
updated_handlers = list(ProxyListener.DEFAULT_LISTENERS) + [MessageModifier()]
monkeypatch.setattr(ProxyListener, "DEFAULT_LISTENERS", updated_handlers)
# prepare bucket and test object
bucket_name = f"b-{short_uid()}"
key = "test/1/2/3"
s3_client.create_bucket(Bucket=bucket_name)
s3_client.put_object(Bucket=bucket_name, Key=key, Body=b"test content 123")
# get content, assert that content has been patched
result = s3_client.get_object(Bucket=bucket_name, Key=key)
content = to_str(result["Body"].read())
assert " patched" in content
def test_update_path_in_url(self):
assert update_path_in_url("http://foo:123", "/bar/1/2/3") == "http://foo:123/bar/1/2/3"
assert update_path_in_url("http://foo:123/", "/bar/1/2/3") == "http://foo:123/bar/1/2/3"
assert (
update_path_in_url("http://foo:123/test", "/bar/1/2/3?p1#h")
== "http://foo:123/bar/1/2/3?p1#h"
)
assert update_path_in_url("http://foo:123/test", "bar/1/2/3") == "http://foo:123/bar/1/2/3"
assert (
update_path_in_url("https://foo:123/test", "bar/1/2/3") == "https://foo:123/bar/1/2/3"
)
assert update_path_in_url("http://foo:123/test", "/") == "http://foo:123/"
assert update_path_in_url("//foo:123/test/123", "bar/1/2/3") == "//foo:123/bar/1/2/3"
def test_response_content_type(self):
url = config.get_edge_url()
data = {"Action": "GetCallerIdentity", "Version": "2011-06-15"}
# receive response as XML (default)
headers = aws_stack.mock_aws_request_headers("sts")
response = requests.post(url, data=data, headers=headers)
assert response
content1 = to_str(response.content)
with pytest.raises(json.decoder.JSONDecodeError):
json.loads(content1)
content1 = xmltodict.parse(content1)
content1_result = content1["GetCallerIdentityResponse"]["GetCallerIdentityResult"]
assert content1_result["Account"] == TEST_AWS_ACCOUNT_ID
# receive response as JSON (via Accept header)
headers = aws_stack.mock_aws_request_headers("sts")
headers["Accept"] = APPLICATION_JSON
response = requests.post(url, data=data, headers=headers)
assert response
content2 = json.loads(to_str(response.content))
content2_result = content2["GetCallerIdentityResponse"]["GetCallerIdentityResult"]
assert content2_result["Account"] == TEST_AWS_ACCOUNT_ID
content1.get("GetCallerIdentityResponse", {}).pop("ResponseMetadata", None)
content2.get("GetCallerIdentityResponse", {}).pop("ResponseMetadata", None)
assert strip_xmlns(content1) == content2
def test_request_with_custom_host_header(self):
url = config.get_edge_url()
headers = aws_stack.mock_aws_request_headers("lambda")
# using a simple for-loop here (instead of pytest parametrization), for simplicity
for host in ["localhost", "example.com"]:
for port in ["", ":123", f":{config.EDGE_PORT}"]:
headers["Host"] = f"{host}{port}"
response = requests.get(f"{url}/2015-03-31/functions", headers=headers)
assert response
assert "Functions" in json.loads(to_str(response.content))
@pytest.mark.skipif(
condition=not config.LEGACY_EDGE_PROXY, reason="only relevant for old edge proxy"
)
def test_forward_raw_path(self, monkeypatch):
class MyListener(ProxyListener):
def forward_request(self, method, path, data, headers):
_path = get_full_raw_path(quart_request)
return {"method": method, "path": _path}
# start listener and configure EDGE_FORWARD_URL
port = get_free_tcp_port()
forward_url = f"http://localhost:{port}"
listener = MyListener()
proxy = start_proxy_server(port, update_listener=listener, use_ssl=True)
monkeypatch.setattr(config, "EDGE_FORWARD_URL", forward_url)
# run test request, assert that raw request path is forwarded
test_arn = "arn:aws:test:resource/test"
raw_path = f"/test/{urllib.parse.quote(test_arn)}/bar?q1=foo&q2=bar"
url = f"{config.get_edge_url()}{raw_path}"
response = requests.get(url)
expected = {"method": "GET", "path": raw_path}
assert json.loads(to_str(response.content)) == expected
proxy.stop()
def run_aws_cmd(service, cmd_params, env=None, cache_duration_secs=None):
cmd = '%s %s' % (aws_cmd(service, env), cmd_params)
if not is_api_enabled(service):
return '{}'
return run_cached(cmd, cache_duration_secs=cache_duration_secs)
