def check_s3(expect_shutdown=False, print_error=False):
out = None
try:
# wait for port to be opened
wait_for_port_open(DEFAULT_PORT_S3_BACKEND)
# check S3
out = aws_stack.connect_to_service(service_name='s3').list_buckets()
except Exception as e:
if print_error:
LOGGER.error('S3 health check failed: %s %s' % (e, traceback.format_exc()))
if expect_shutdown:
assert out is None
else:
assert isinstance(out['Buckets'], list)
def check_dynamodb(expect_shutdown=False, print_error=False):
out = None
try:
# wait for port to be opened
wait_for_port_open(DEFAULT_PORT_DYNAMODB_BACKEND)
# check DynamoDB
out = aws_stack.connect_to_service(service_name='dynamodb').list_tables()
except Exception as e:
if print_error:
LOGGER.error('DynamoDB health check failed: %s %s' % (e, traceback.format_exc()))
if expect_shutdown:
assert out is None
else:
assert isinstance(out['TableNames'], list)
def test_redrive_policy_http_subscription(self):
self.unsubscribe_all_from_sns()
# create HTTP endpoint and connect it to SNS topic
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
records.append((json.loads(to_str(data)), headers))
return 200
records = []
local_port = get_free_tcp_port()
proxy = start_proxy(local_port,
backend_url=None,
update_listener=MyUpdateListener())
wait_for_port_open(local_port)
http_endpoint = '%s://localhost:%s' % (get_service_protocol(),
local_port)
subscription = self.sns_client.subscribe(TopicArn=self.topic_arn,
Protocol='http',
Endpoint=http_endpoint)
self.sns_client.set_subscription_attributes(
SubscriptionArn=subscription['SubscriptionArn'],
AttributeName='RedrivePolicy',
AttributeValue=json.dumps({
'deadLetterTargetArn':
aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME)
}))
proxy.stop()
self.sns_client.publish(TopicArn=self.topic_arn,
Message=json.dumps(
{'message': 'test_redrive_policy'}))
def receive_dlq():
result = self.sqs_client.receive_message(
QueueUrl=self.dlq_url, MessageAttributeNames=['All'])
self.assertGreater(len(result['Messages']), 0)
self.assertEqual(
json.loads(
json.loads(result['Messages'][0]['Body'])['Message'][0])
['message'], 'test_redrive_policy')
retry(receive_dlq, retries=10, sleep=2)
def start_kms_local(port=None, backend_port=None, asynchronous=None, update_listener=None):
port = port or config.PORT_KMS
backend_port = get_free_tcp_port()
kms_binary = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", get_os())
log_startup_message("KMS")
start_proxy_for_service("kms", port, backend_port, update_listener)
env_vars = {
"PORT": str(backend_port),
"KMS_REGION": config.DEFAULT_REGION,
"REGION": config.DEFAULT_REGION,
"KMS_ACCOUNT_ID": TEST_AWS_ACCOUNT_ID,
"ACCOUNT_ID": TEST_AWS_ACCOUNT_ID,
}
if config.dirs.data:
env_vars["KMS_DATA_PATH"] = config.dirs.data
result = do_run(kms_binary, asynchronous, env_vars=env_vars)
wait_for_port_open(backend_port)
return result
def start_kms(port=None, backend_port=None, asynchronous=None, update_listener=None):
port = port or config.PORT_KMS
backend_port = get_free_tcp_port()
kms_binary = INSTALL_PATH_KMS_BINARY_PATTERN.replace('<arch>', get_arch())
log_startup_message('KMS')
start_proxy_for_service('kms', port, backend_port, update_listener)
env_vars = {
'PORT': str(backend_port),
'KMS_REGION': config.DEFAULT_REGION,
'REGION': config.DEFAULT_REGION,
'KMS_ACCOUNT_ID': TEST_AWS_ACCOUNT_ID,
'ACCOUNT_ID': TEST_AWS_ACCOUNT_ID
}
if config.DATA_DIR:
env_vars['KMS_DATA_PATH'] = config.DATA_DIR
result = do_run(kms_binary, asynchronous, env_vars=env_vars)
wait_for_port_open(backend_port)
return result
def check_dynamodb(expect_shutdown=False, print_error=False):
out = None
try:
# wait for backend port to be opened
wait_for_port_open(PORT_DYNAMODB_BACKEND,
http_path='/',
expect_success=False,
sleep_time=1)
# check DynamoDB
out = aws_stack.connect_to_service('dynamodb').list_tables()
except Exception as e:
if print_error:
LOGGER.error('DynamoDB health check failed: %s %s' %
(e, traceback.format_exc()))
if expect_shutdown:
assert out is None
else:
assert isinstance(out['TableNames'], list)
def start_kms(port=None,
backend_port=None,
asynchronous=None,
update_listener=None):
port = port or config.PORT_KMS
backend_port = get_free_tcp_port()
kms_binary = INSTALL_PATH_KMS_BINARY_PATTERN.replace('<arch>', get_arch())
print('Starting mock KMS service on %s ...' % edge_ports_info())
start_proxy_for_service('kms', port, backend_port, update_listener)
env_vars = {
'PORT': str(backend_port),
'KMS_REGION': config.DEFAULT_REGION,
'REGION': config.DEFAULT_REGION,
'KMS_ACCOUNT_ID': TEST_AWS_ACCOUNT_ID,
'ACCOUNT_ID': TEST_AWS_ACCOUNT_ID
}
result = do_run(kms_binary, asynchronous, env_vars=env_vars)
wait_for_port_open(backend_port)
return result
def start_server_process(port):
if '__server__' in API_SERVERS:
return API_SERVERS['__server__']['thread']
port = port or MULTI_SERVER_PORT
API_SERVERS['__server__'] = config = {'port': port}
LOG.info('Starting multi API server process on port %s' % port)
if RUN_SERVER_IN_PROCESS:
cmd = '"%s" "%s" %s' % (sys.executable, __file__, port)
env_vars = {
'PYTHONPATH': '.:%s' % constants.LOCALSTACK_ROOT_FOLDER
}
thread = ShellCommandThread(cmd, outfile=subprocess.PIPE, env_vars=env_vars, inherit_cwd=True)
thread.start()
else:
thread = start_server(port, asynchronous=True)
TMP_THREADS.append(thread)
config['thread'] = thread
wait_for_port_open(port, retries=20, sleep_time=1)
return thread
def test_subscribe_http_endpoint(self):
# create HTTP endpoint and connect it to SNS topic
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
records.append((json.loads(to_str(data)), headers))
return 200
records = []
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener())
wait_for_port_open(local_port)
queue_arn = '%s://localhost:%s' % (get_service_protocol(), local_port)
self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol='http', Endpoint=queue_arn)
def received():
assert records[0][0]['Type'] == 'SubscriptionConfirmation'
assert records[0][1]['x-amz-sns-message-type'] == 'SubscriptionConfirmation'
retry(received, retries=5, sleep=1)
proxy.stop()
def check_lambda(expect_shutdown=False, print_error=False):
out = None
try:
from localstack.services.infra import PROXY_LISTENERS
from localstack.utils.aws import aws_stack
from localstack.utils.common import wait_for_port_open
# wait for port to be opened
port = PROXY_LISTENERS.get("lambda")[1]
wait_for_port_open(port) # TODO get lambda port in a cleaner way
endpoint_url = f"http://127.0.0.1:{port}"
out = aws_stack.connect_to_service(
service_name="lambda", endpoint_url=endpoint_url).list_functions()
except Exception:
if print_error:
LOG.exception("Lambda health check failed")
if expect_shutdown:
assert out is None
else:
assert out and isinstance(out.get("Functions"), list)
def start_server_process(port):
if "__server__" in API_SERVERS:
return API_SERVERS["__server__"]["thread"]
port = port or get_multi_server_port()
API_SERVERS["__server__"] = config = {"port": port}
LOG.info("Starting multi API server process on port %s" % port)
if RUN_SERVER_IN_PROCESS:
cmd = '"%s" "%s" %s' % (sys.executable, __file__, port)
env_vars = {"PYTHONPATH": ".:%s" % constants.LOCALSTACK_ROOT_FOLDER}
thread = ShellCommandThread(cmd,
outfile=subprocess.PIPE,
env_vars=env_vars,
inherit_cwd=True)
thread.start()
else:
thread = start_server(port, asynchronous=True)
TMP_THREADS.append(thread)
config["thread"] = thread
wait_for_port_open(port, retries=20, sleep_time=1)
return thread
def test_ssl_proxy_server():
class MyListener(ProxyListener):
def forward_request(self, *args, **kwargs):
invocations.append((args, kwargs))
return {"foo": "bar"}
invocations = []
# start SSL proxy
listener = MyListener()
port = get_free_tcp_port()
server = start_proxy_server(port, update_listener=listener, use_ssl=True)
wait_for_port_open(port)
# start SSL proxy
proxy_port = get_free_tcp_port()
proxy = start_ssl_proxy(proxy_port, port, asynchronous=True, fix_encoding=True)
wait_for_port_open(proxy_port)
# invoke SSL proxy server
url = f"https://{LOCALHOST_HOSTNAME}:{proxy_port}"
num_requests = 3
for i in range(num_requests):
response = requests.get(url, verify=False)
assert response.status_code == 200
# assert backend server has been invoked
assert len(invocations) == num_requests
# invoke SSL proxy server with gzip response
for encoding in ["gzip", "gzip, deflate"]:
headers = {HEADER_ACCEPT_ENCODING: encoding}
response = requests.get(url, headers=headers, verify=False, stream=True)
result = response.raw.read()
assert to_str(gzip.decompress(result)) == json.dumps({"foo": "bar"})
# clean up
proxy.stop()
server.stop()
def check_secretsmanager(expect_shutdown=False, print_error=False):
out = None
# noinspection PyBroadException
try:
wait_for_port_open(PORT_SECRETS_MANAGER_BACKEND,
http_path="/",
expect_success=False)
endpoint_url = f"http://127.0.0.1:{PORT_SECRETS_MANAGER_BACKEND}"
out = aws_stack.connect_to_service(
service_name="secretsmanager",
endpoint_url=endpoint_url).list_secrets()
except Exception:
if print_error:
logger = logging.getLogger(__name__)
logger.exception("Secretsmanager health check failed")
if expect_shutdown:
assert out is None
return
assert isinstance(out["SecretList"], list)
def test_static_route(self):
class MyListener(ProxyListener):
def forward_request(self, method, path, *args, **kwargs):
return {"method": method, "path": path}
# start proxy server
listener = MyListener()
port = get_free_tcp_port()
server = start_proxy_server(port, update_listener=listener)
wait_for_port_open(port)
# request a /static/... path from the server and assert result
url = f"http://{LOCALHOST_HOSTNAME}:{port}/static/index.html"
response = requests.get(url, verify=False)
assert response.ok
assert json.loads(to_str(response.content)) == {
"method": "GET",
"path": "/static/index.html",
}
# clean up
server.stop()
def check_kinesis(expect_shutdown=False, print_error=False):
if expect_shutdown is False and kinesis_stopped.is_set():
raise AssertionError("kinesis backend has stopped")
out = None
try:
# check Kinesis
wait_for_port_open(PORT_KINESIS_BACKEND,
http_path="/",
expect_success=False,
sleep_time=1)
endpoint_url = f"http://127.0.0.1:{PORT_KINESIS_BACKEND}"
out = aws_stack.connect_to_service(
service_name="kinesis", endpoint_url=endpoint_url).list_streams()
except Exception:
if print_error:
LOGGER.exception("Kinesis health check failed")
if expect_shutdown:
assert out is None or kinesis_stopped.is_set()
else:
assert not kinesis_stopped.is_set()
assert out and isinstance(out.get("StreamNames"), list)
def test_subscribe_http_endpoint(self):
# create HTTP endpoint and connect it to SNS topic
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
records.append((json.loads(to_str(data)), headers))
return 200
records = []
local_port = get_free_tcp_port()
proxy = start_proxy(local_port,
backend_url=None,
update_listener=MyUpdateListener())
wait_for_port_open(local_port)
queue_arn = "%s://localhost:%s" % (get_service_protocol(), local_port)
self.sns_client.subscribe(TopicArn=self.topic_arn,
Protocol="http",
Endpoint=queue_arn)
def received():
self.assertEqual(records[0][0]["Type"], "SubscriptionConfirmation")
self.assertEqual(records[0][1]["x-amz-sns-message-type"],
"SubscriptionConfirmation")
token = records[0][0]["Token"]
subscribe_url = records[0][0]["SubscribeURL"]
self.assertEqual(
subscribe_url,
"%s/?Action=ConfirmSubscription&TopicArn=%s&Token=%s" %
(external_service_url("sns"), self.topic_arn, token),
)
self.assertIn("Signature", records[0][0])
self.assertIn("SigningCertURL", records[0][0])
retry(received, retries=5, sleep=1)
proxy.stop()
def test_multiple_subscriptions_http_endpoint(self):
self.unsubscribe_all_from_sns()
# create HTTP endpoint and connect it to SNS topic
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
records.append((json.loads(to_str(data)), headers))
return 429
number_of_subscriptions = 4
records = []
proxies = []
for _ in range(number_of_subscriptions):
local_port = get_free_tcp_port()
proxies.append(
start_proxy(local_port,
backend_url=None,
update_listener=MyUpdateListener()))
wait_for_port_open(local_port)
http_endpoint = "%s://localhost:%s" % (get_service_protocol(),
local_port)
self.sns_client.subscribe(TopicArn=self.topic_arn,
Protocol="http",
Endpoint=http_endpoint)
# fetch subscription information
subscription_list = self.sns_client.list_subscriptions()
self.assertEqual(
subscription_list["ResponseMetadata"]["HTTPStatusCode"], 200)
self.assertEqual(len(subscription_list["Subscriptions"]),
number_of_subscriptions)
self.assertEqual(number_of_subscriptions, len(records))
for proxy in proxies:
proxy.stop()
def test_scheduled_expression_events(self):
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
return 200
local_port = get_free_tcp_port()
proxy = start_proxy(local_port,
backend_url=None,
update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
topic_name = 'topic-{}'.format(short_uid())
queue_name = 'queue-{}'.format(short_uid())
rule_name = 'rule-{}'.format(short_uid())
endpoint = '{}://{}:{}'.format(get_service_protocol(),
config.LOCALSTACK_HOSTNAME, local_port)
sm_role_arn = aws_stack.role_arn('sfn_role')
sm_name = 'state-machine-{}'.format(short_uid())
topic_target_id = 'target-{}'.format(short_uid())
sm_target_id = 'target-{}'.format(short_uid())
queue_target_id = 'target-{}'.format(short_uid())
events = []
state_machine_definition = """
{
"StartAt": "Hello",
"States": {
"Hello": {
"Type": "Pass",
"Result": "World",
"End": true
}
}
}
"""
state_machine_arn = self.sfn_client.create_state_machine(
name=sm_name,
definition=state_machine_definition,
roleArn=sm_role_arn)['stateMachineArn']
topic_arn = self.sns_client.create_topic(Name=topic_name)['TopicArn']
self.sns_client.subscribe(TopicArn=topic_arn,
Protocol='http',
Endpoint=endpoint)
queue_url = self.sqs_client.create_queue(
QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
event = {'env': 'testing'}
self.events_client.put_rule(Name=rule_name,
ScheduleExpression='rate(1 minutes)')
self.events_client.put_targets(Rule=rule_name,
Targets=[{
'Id': topic_target_id,
'Arn': topic_arn,
'Input': json.dumps(event)
}, {
'Id': sm_target_id,
'Arn': state_machine_arn,
'Input': json.dumps(event)
}, {
'Id': queue_target_id,
'Arn': queue_arn,
'Input': json.dumps(event)
}])
def received(q_url):
# state machine got executed
executions = self.sfn_client.list_executions(
stateMachineArn=state_machine_arn)['executions']
self.assertGreaterEqual(len(executions), 1)
# http endpoint got events
self.assertGreaterEqual(len(events), 2)
notifications = [
event['Message'] for event in events
if event['Type'] == 'Notification'
]
self.assertGreaterEqual(len(notifications), 1)
# get state machine execution detail
execution_arn = executions[0]['executionArn']
execution_input = self.sfn_client.describe_execution(
executionArn=execution_arn)['input']
# get message from queue
msgs = self.sqs_client.receive_message(QueueUrl=q_url).get(
'Messages', [])
self.assertGreaterEqual(len(msgs), 1)
return execution_input, notifications[0], msgs[0]
execution_input, notification, msg_received = retry(received,
retries=5,
sleep=15,
q_url=queue_url)
self.assertEqual(json.loads(notification), event)
self.assertEqual(json.loads(execution_input), event)
self.assertEqual(json.loads(msg_received['Body']), event)
proxy.stop()
self.events_client.remove_targets(Rule=rule_name,
Ids=[topic_target_id, sm_target_id],
Force=True)
self.events_client.delete_rule(Name=rule_name, Force=True)
self.sns_client.delete_topic(TopicArn=topic_arn)
self.sfn_client.delete_state_machine(stateMachineArn=state_machine_arn)
self.sqs_client.delete_queue(QueueUrl=queue_url)
def test_api_destinations(self, events_client):
token = short_uid()
bearer = "Bearer %s" % token
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
paths_list.append(path)
auth = headers.get("Api") or headers.get("Authorization")
if auth not in headers_list:
headers_list.append(auth)
if headers.get("target_header"):
headers_list.append(headers.get("target_header"))
if "client_id" in event:
oauth_data.update(
{
"client_id": event.get("client_id"),
"client_secret": event.get("client_secret"),
"header_value": headers.get("oauthheader"),
"body_value": event.get("oauthbody"),
"path": path,
}
)
return requests_response(
{
"access_token": token,
"token_type": "Bearer",
"expires_in": 86400,
}
)
events = []
paths_list = []
headers_list = []
oauth_data = {}
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
url = f"http://localhost:{local_port}"
auth_types = [
{
"type": "BASIC",
"key": "BasicAuthParameters",
"parameters": {"Username": "******", "Password": "******"},
},
{
"type": "API_KEY",
"key": "ApiKeyAuthParameters",
"parameters": {"ApiKeyName": "Api", "ApiKeyValue": "apikey_secret"},
},
{
"type": "OAUTH_CLIENT_CREDENTIALS",
"key": "OAuthParameters",
"parameters": {
"AuthorizationEndpoint": url,
"ClientParameters": {"ClientID": "id", "ClientSecret": "password"},
"HttpMethod": "put",
"OAuthHttpParameters": {
"BodyParameters": [{"Key": "oauthbody", "Value": "value1"}],
"HeaderParameters": [{"Key": "oauthheader", "Value": "value2"}],
"QueryStringParameters": [{"Key": "oauthquery", "Value": "value3"}],
},
},
},
]
for auth in auth_types:
connection_name = "c-%s" % short_uid()
connection_arn = events_client.create_connection(
Name=connection_name,
AuthorizationType=auth.get("type"),
AuthParameters={
auth.get("key"): auth.get("parameters"),
"InvocationHttpParameters": {
"BodyParameters": [
{"Key": "key", "Value": "value", "IsValueSecret": False}
],
"HeaderParameters": [
{"Key": "key", "Value": "value", "IsValueSecret": False}
],
"QueryStringParameters": [
{"Key": "key", "Value": "value", "IsValueSecret": False}
],
},
},
)["ConnectionArn"]
# create api destination
dest_name = "d-%s" % short_uid()
result = events_client.create_api_destination(
Name=dest_name,
ConnectionArn=connection_arn,
InvocationEndpoint=url,
HttpMethod="POST",
)
# create rule and target
rule_name = "r-%s" % short_uid()
target_id = "target-{}".format(short_uid())
pattern = json.dumps({"source": ["source-123"], "detail-type": ["type-123"]})
events_client.put_rule(Name=rule_name, EventPattern=pattern)
events_client.put_targets(
Rule=rule_name,
Targets=[
{
"Id": target_id,
"Arn": result["ApiDestinationArn"],
"Input": '{"target_value":"value"}',
"HttpParameters": {
"PathParameterValues": ["target_path"],
"HeaderParameters": {"target_header": "target_header_value"},
"QueryStringParameters": {"target_query": "t_query"},
},
}
],
)
entries = [
{
"Source": "source-123",
"DetailType": "type-123",
"Detail": '{"i": %s}' % 0,
}
]
events_client.put_events(Entries=entries)
# cleaning
events_client.delete_connection(Name=connection_name)
events_client.delete_api_destination(Name=dest_name)
events_client.delete_rule(Name=rule_name, Force=True)
# assert that all events have been received in the HTTP server listener
def check():
assert len(events) >= len(auth_types)
assert "key" in paths_list[0] and "value" in paths_list[0]
assert "target_query" in paths_list[0] and "t_query" in paths_list[0]
assert "target_path" in paths_list[0]
assert events[0].get("key") == "value"
assert events[0].get("target_value") == "value"
assert oauth_data.get("client_id") == "id"
assert oauth_data.get("client_secret") == "password"
assert oauth_data.get("header_value") == "value2"
assert oauth_data.get("body_value") == "value1"
assert "oauthquery" in oauth_data.get("path")
assert "value3" in oauth_data.get("path")
user_pass = to_str(base64.b64encode(b"user:pass"))
assert f"Basic {user_pass}" in headers_list
assert "apikey_secret" in headers_list
assert bearer in headers_list
assert "target_header_value" in headers_list
retry(check, sleep=0.5, retries=5)
# clean up
proxy.stop()
def test_scheduled_expression_events(
self, stepfunctions_client, sns_client, sqs_client, events_client
):
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
return 200
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
topic_name = "topic-{}".format(short_uid())
queue_name = "queue-{}".format(short_uid())
fifo_queue_name = "queue-{}.fifo".format(short_uid())
rule_name = "rule-{}".format(short_uid())
endpoint = "{}://{}:{}".format(
get_service_protocol(), config.LOCALSTACK_HOSTNAME, local_port
)
sm_role_arn = aws_stack.role_arn("sfn_role")
sm_name = "state-machine-{}".format(short_uid())
topic_target_id = "target-{}".format(short_uid())
sm_target_id = "target-{}".format(short_uid())
queue_target_id = "target-{}".format(short_uid())
fifo_queue_target_id = "target-{}".format(short_uid())
events = []
state_machine_definition = """
{
"StartAt": "Hello",
"States": {
"Hello": {
"Type": "Pass",
"Result": "World",
"End": true
}
}
}
"""
state_machine_arn = stepfunctions_client.create_state_machine(
name=sm_name, definition=state_machine_definition, roleArn=sm_role_arn
)["stateMachineArn"]
topic_arn = sns_client.create_topic(Name=topic_name)["TopicArn"]
sns_client.subscribe(TopicArn=topic_arn, Protocol="http", Endpoint=endpoint)
queue_url = sqs_client.create_queue(QueueName=queue_name)["QueueUrl"]
fifo_queue_url = sqs_client.create_queue(
QueueName=fifo_queue_name,
Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"},
)["QueueUrl"]
queue_arn = aws_stack.sqs_queue_arn(queue_name)
fifo_queue_arn = aws_stack.sqs_queue_arn(fifo_queue_name)
event = {"env": "testing"}
events_client.put_rule(Name=rule_name, ScheduleExpression="rate(1 minutes)")
events_client.put_targets(
Rule=rule_name,
Targets=[
{"Id": topic_target_id, "Arn": topic_arn, "Input": json.dumps(event)},
{
"Id": sm_target_id,
"Arn": state_machine_arn,
"Input": json.dumps(event),
},
{"Id": queue_target_id, "Arn": queue_arn, "Input": json.dumps(event)},
{
"Id": fifo_queue_target_id,
"Arn": fifo_queue_arn,
"Input": json.dumps(event),
"SqsParameters": {"MessageGroupId": "123"},
},
],
)
def received(q_urls):
# state machine got executed
executions = stepfunctions_client.list_executions(stateMachineArn=state_machine_arn)[
"executions"
]
assert len(executions) >= 1
# http endpoint got events
assert len(events) >= 2
notifications = [
event["Message"] for event in events if event["Type"] == "Notification"
]
assert len(notifications) >= 1
# get state machine execution detail
execution_arn = executions[0]["executionArn"]
execution_input = stepfunctions_client.describe_execution(executionArn=execution_arn)[
"input"
]
all_msgs = []
# get message from queue
for url in q_urls:
msgs = sqs_client.receive_message(QueueUrl=url).get("Messages", [])
assert len(msgs) >= 1
all_msgs.append(msgs[0])
return execution_input, notifications[0], all_msgs
execution_input, notification, msgs_received = retry(
received, retries=5, sleep=15, q_urls=[queue_url, fifo_queue_url]
)
assert json.loads(notification) == event
assert json.loads(execution_input) == event
for msg_received in msgs_received:
assert json.loads(msg_received["Body"]) == event
# clean up
proxy.stop()
self.cleanup(
None, rule_name, target_ids=[topic_target_id, sm_target_id], queue_url=queue_url
)
sns_client.delete_topic(TopicArn=topic_arn)
stepfunctions_client.delete_state_machine(stateMachineArn=state_machine_arn)
def test_firehose_http(lambda_processor_enabled: bool):
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
data_received = dict(json.loads(data.decode("utf-8")))
records.append(data_received)
return 200
if lambda_processor_enabled:
# create processor func
func_name = f"proc-{short_uid()}"
testutil.create_lambda_function(handler_file=PROCESSOR_LAMBDA,
func_name=func_name)
# define firehose configs
local_port = get_free_tcp_port()
endpoint = "{}://{}:{}".format(get_service_protocol(),
config.LOCALSTACK_HOSTNAME, local_port)
records = []
http_destination_update = {
"EndpointConfiguration": {
"Url": endpoint,
"Name": "test_update"
}
}
http_destination = {
"EndpointConfiguration": {
"Url": endpoint
},
"S3BackupMode": "FailedDataOnly",
"S3Configuration": {
"RoleARN": "arn:.*",
"BucketARN": "arn:.*",
"Prefix": "",
"ErrorOutputPrefix": "",
"BufferingHints": {
"SizeInMBs": 1,
"IntervalInSeconds": 60
},
},
}
if lambda_processor_enabled:
http_destination["ProcessingConfiguration"] = {
"Enabled":
True,
"Processors": [{
"Type":
"Lambda",
"Parameters": [{
"ParameterName":
"LambdaArn",
"ParameterValue":
lambda_function_arn(func_name),
}],
}],
}
# start proxy server
start_proxy(local_port,
backend_url=None,
update_listener=MyUpdateListener())
wait_for_port_open(local_port)
# create firehose stream with http destination
firehose = aws_stack.create_external_boto_client("firehose")
stream_name = "firehose_" + short_uid()
stream = firehose.create_delivery_stream(
DeliveryStreamName=stream_name,
HttpEndpointDestinationConfiguration=http_destination,
)
assert stream
stream_description = firehose.describe_delivery_stream(
DeliveryStreamName=stream_name)
stream_description = stream_description["DeliveryStreamDescription"]
destination_description = stream_description["Destinations"][0][
"HttpEndpointDestinationDescription"]
assert len(stream_description["Destinations"]) == 1
assert (destination_description["EndpointConfiguration"]["Url"] ==
f"http://localhost:{local_port}")
# put record
msg_text = "Hello World!"
firehose.put_record(DeliveryStreamName=stream_name,
Record={"Data": msg_text})
# wait for the result to arrive with proper content
def _assert_record():
received_record = records[0]["records"][0]
received_record_data = to_str(
base64.b64decode(to_bytes(received_record["data"])))
assert (
received_record_data ==
f"{msg_text}{'-processed' if lambda_processor_enabled else ''}")
retry(_assert_record, retries=5, sleep=1)
# update stream destination
destination_id = stream_description["Destinations"][0]["DestinationId"]
version_id = stream_description["VersionId"]
firehose.update_destination(
DeliveryStreamName=stream_name,
DestinationId=destination_id,
CurrentDeliveryStreamVersionId=version_id,
HttpEndpointDestinationUpdate=http_destination_update,
)
stream_description = firehose.describe_delivery_stream(
DeliveryStreamName=stream_name)
stream_description = stream_description["DeliveryStreamDescription"]
destination_description = stream_description["Destinations"][0][
"HttpEndpointDestinationDescription"]
assert destination_description["EndpointConfiguration"][
"Name"] == "test_update"
# delete stream
stream = firehose.delete_delivery_stream(DeliveryStreamName=stream_name)
assert stream["ResponseMetadata"]["HTTPStatusCode"] == 200
def test_firehose_http(self):
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
data_received = dict(json.loads(data.decode('utf-8')))
records.append(data_received)
return 200
firehose = aws_stack.connect_to_service('firehose')
local_port = get_free_tcp_port()
endpoint = '{}://{}:{}'.format(get_service_protocol(),
config.LOCALSTACK_HOSTNAME, local_port)
records = []
http_destination_update = {
'EndpointConfiguration': {
'Url': endpoint,
'Name': 'test_update'
}
}
http_destination = {
'EndpointConfiguration': {
'Url': endpoint
},
'S3BackupMode': 'FailedDataOnly',
'S3Configuration': {
'RoleARN': 'arn:.*',
'BucketARN': 'arn:.*',
'Prefix': '',
'ErrorOutputPrefix': '',
'BufferingHints': {
'SizeInMBs': 1,
'IntervalInSeconds': 60
}
}
}
# start proxy server
start_proxy(local_port,
backend_url=None,
update_listener=MyUpdateListener())
wait_for_port_open(local_port)
# create firehose stream with http destination
stream = firehose.create_delivery_stream(
DeliveryStreamName=TEST_STREAM_NAME,
HttpEndpointDestinationConfiguration=http_destination)
self.assertTrue(stream)
stream_description = firehose.describe_delivery_stream(
DeliveryStreamName=TEST_STREAM_NAME)
stream_description = stream_description['DeliveryStreamDescription']
destination_description = stream_description['Destinations'][0][
'HttpEndpointDestinationDescription']
self.assertEquals(1, len(stream_description['Destinations']))
self.assertEquals(
f'http://localhost:{local_port}',
destination_description['EndpointConfiguration']['Url'])
# put record
firehose.put_record(DeliveryStreamName=TEST_STREAM_NAME,
Record={'Data': 'Hello World!'})
record_received = to_str(
base64.b64decode(to_bytes(records[0]['records'][0]['data'])))
# wait for the result to arrive with proper content
retry(lambda: self.assertEquals('Hello World!', record_received),
retries=5,
sleep=1)
# update stream destination
destination_id = stream_description['Destinations'][0]['DestinationId']
version_id = stream_description['VersionId']
firehose.update_destination(
DeliveryStreamName=TEST_STREAM_NAME,
DestinationId=destination_id,
CurrentDeliveryStreamVersionId=version_id,
HttpEndpointDestinationUpdate=http_destination_update)
stream_description = firehose.describe_delivery_stream(
DeliveryStreamName=TEST_STREAM_NAME)
stream_description = stream_description['DeliveryStreamDescription']
destination_description = stream_description['Destinations'][0][
'HttpEndpointDestinationDescription']
self.assertEquals(
'test_update',
destination_description['EndpointConfiguration']['Name'])
# delete stream
stream = firehose.delete_delivery_stream(
DeliveryStreamName=TEST_STREAM_NAME)
self.assertEquals(200, stream['ResponseMetadata']['HTTPStatusCode'])
def test_api_destinations(self):
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
return 200
events = []
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
events_client = aws_stack.connect_to_service("events")
connection_arn = events_client.create_connection(
Name="TestConnection",
AuthorizationType="BASIC",
AuthParameters={
"BasicAuthParameters": {
"Username": "******",
"Password": "******"
}
},
)["ConnectionArn"]
# create api destination
dest_name = "d-%s" % short_uid()
url = "http://localhost:%s" % local_port
result = self.events_client.create_api_destination(
Name=dest_name,
ConnectionArn=connection_arn,
InvocationEndpoint=url,
HttpMethod="POST",
)
# create rule and target
rule_name = "r-%s" % short_uid()
target_id = "target-{}".format(short_uid())
pattern = json.dumps({
"source": ["source-123"],
"detail-type": ["type-123"]
})
self.events_client.put_rule(Name=rule_name, EventPattern=pattern)
self.events_client.put_targets(
Rule=rule_name,
Targets=[{
"Id": target_id,
"Arn": result["ApiDestinationArn"]
}],
)
# put events, to trigger rules
num_events = 5
for i in range(num_events):
entries = [{
"Source": "source-123",
"DetailType": "type-123",
"Detail": '{"i": %s}' % i,
}]
self.events_client.put_events(Entries=entries)
# assert that all events have been received in the HTTP server listener
def check():
self.assertEqual(len(events), num_events)
retry(check, sleep=0.5, retries=5)
# clean up
proxy.stop()
def test_firehose_http(self):
class MyUpdateListener(ProxyListener):
def forward_request(self, method, path, data, headers):
data_received = dict(json.loads(data.decode("utf-8")))
records.append(data_received)
return 200
firehose = aws_stack.connect_to_service("firehose")
local_port = get_free_tcp_port()
endpoint = "{}://{}:{}".format(
get_service_protocol(), config.LOCALSTACK_HOSTNAME, local_port
)
records = []
http_destination_update = {
"EndpointConfiguration": {"Url": endpoint, "Name": "test_update"}
}
http_destination = {
"EndpointConfiguration": {"Url": endpoint},
"S3BackupMode": "FailedDataOnly",
"S3Configuration": {
"RoleARN": "arn:.*",
"BucketARN": "arn:.*",
"Prefix": "",
"ErrorOutputPrefix": "",
"BufferingHints": {"SizeInMBs": 1, "IntervalInSeconds": 60},
},
}
# start proxy server
start_proxy(local_port, backend_url=None, update_listener=MyUpdateListener())
wait_for_port_open(local_port)
# create firehose stream with http destination
stream = firehose.create_delivery_stream(
DeliveryStreamName=TEST_STREAM_NAME,
HttpEndpointDestinationConfiguration=http_destination,
)
self.assertTrue(stream)
stream_description = firehose.describe_delivery_stream(DeliveryStreamName=TEST_STREAM_NAME)
stream_description = stream_description["DeliveryStreamDescription"]
destination_description = stream_description["Destinations"][0][
"HttpEndpointDestinationDescription"
]
self.assertEqual(1, len(stream_description["Destinations"]))
self.assertEqual(
f"http://localhost:{local_port}",
destination_description["EndpointConfiguration"]["Url"],
)
# put record
firehose.put_record(DeliveryStreamName=TEST_STREAM_NAME, Record={"Data": "Hello World!"})
record_received = to_str(base64.b64decode(to_bytes(records[0]["records"][0]["data"])))
# wait for the result to arrive with proper content
retry(
lambda: self.assertEqual("Hello World!", record_received),
retries=5,
sleep=1,
)
# update stream destination
destination_id = stream_description["Destinations"][0]["DestinationId"]
version_id = stream_description["VersionId"]
firehose.update_destination(
DeliveryStreamName=TEST_STREAM_NAME,
DestinationId=destination_id,
CurrentDeliveryStreamVersionId=version_id,
HttpEndpointDestinationUpdate=http_destination_update,
)
stream_description = firehose.describe_delivery_stream(DeliveryStreamName=TEST_STREAM_NAME)
stream_description = stream_description["DeliveryStreamDescription"]
destination_description = stream_description["Destinations"][0][
"HttpEndpointDestinationDescription"
]
self.assertEqual("test_update", destination_description["EndpointConfiguration"]["Name"])
# delete stream
stream = firehose.delete_delivery_stream(DeliveryStreamName=TEST_STREAM_NAME)
self.assertEqual(200, stream["ResponseMetadata"]["HTTPStatusCode"])
def test_api_destinations(self):
token = short_uid()
bearer = "Bearer %s" % token
class HttpEndpointListener(ProxyListener):
def forward_request(self, method, path, data, headers):
event = json.loads(to_str(data))
events.append(event)
paths_list.append(path)
auth = headers.get("Api") or headers.get("Authorization")
if auth not in headers_list:
headers_list.append(auth)
return requests_response({
"access_token": token,
"token_type": "Bearer",
"expires_in": 86400,
})
events = []
paths_list = []
headers_list = []
local_port = get_free_tcp_port()
proxy = start_proxy(local_port, update_listener=HttpEndpointListener())
wait_for_port_open(local_port)
events_client = aws_stack.create_external_boto_client("events")
url = "http://localhost:%s" % local_port
auth_types = [
{
"type": "BASIC",
"key": "BasicAuthParameters",
"parameters": {
"Username": "******",
"Password": "******"
},
},
{
"type": "API_KEY",
"key": "ApiKeyAuthParameters",
"parameters": {
"ApiKeyName": "Api",
"ApiKeyValue": "apikey_secret"
},
},
{
"type": "OAUTH_CLIENT_CREDENTIALS",
"key": "OAuthParameters",
"parameters": {
"AuthorizationEndpoint": url,
"ClientParameters": {
"ClientID": "id",
"ClientSecret": "password"
},
"HttpMethod": "put",
},
},
]
for auth in auth_types:
connection_name = "c-%s" % short_uid()
connection_arn = events_client.create_connection(
Name=connection_name,
AuthorizationType=auth.get("type"),
AuthParameters={
auth.get("key"): auth.get("parameters"),
"InvocationHttpParameters": {
"BodyParameters": [{
"Key": "key",
"Value": "value",
"IsValueSecret": False
}],
"HeaderParameters": [{
"Key": "key",
"Value": "value",
"IsValueSecret": False
}],
"QueryStringParameters": [{
"Key": "key",
"Value": "value",
"IsValueSecret": False
}],
},
},
)["ConnectionArn"]
# create api destination
dest_name = "d-%s" % short_uid()
result = self.events_client.create_api_destination(
Name=dest_name,
ConnectionArn=connection_arn,
InvocationEndpoint=url,
HttpMethod="POST",
)
# create rule and target
rule_name = "r-%s" % short_uid()
target_id = "target-{}".format(short_uid())
pattern = json.dumps({
"source": ["source-123"],
"detail-type": ["type-123"]
})
self.events_client.put_rule(Name=rule_name, EventPattern=pattern)
self.events_client.put_targets(
Rule=rule_name,
Targets=[{
"Id": target_id,
"Arn": result["ApiDestinationArn"]
}],
)
entries = [{
"Source": "source-123",
"DetailType": "type-123",
"Detail": '{"i": %s}' % 0,
}]
self.events_client.put_events(Entries=entries)
# cleaning
self.events_client.delete_connection(Name=connection_name)
self.events_client.delete_api_destination(Name=dest_name)
self.events_client.delete_rule(Name=rule_name, Force=True)
# assert that all events have been received in the HTTP server listener
def check():
self.assertTrue(len(events) >= len(auth_types))
self.assertTrue("key" in paths_list[0]
and "value" in paths_list[0])
self.assertTrue(events[0].get("key") == "value")
# TODO examine behavior difference between LS pro/community
# Pro seems to (correctly) use base64 for basic authentication instead of plaintext
user_pass = to_str(base64.b64encode(b"user:pass"))
self.assertTrue("Basic user:pass" in headers_list
or f"Basic {user_pass}" in headers_list)
self.assertTrue("apikey_secret" in headers_list)
self.assertTrue(bearer in headers_list)
retry(check, sleep=0.5, retries=5)
# clean up
proxy.stop()
