def test_intrinsic_functions(self):
state_machines_before = self.sfn_client.list_state_machines(
)["stateMachines"]
# create state machine
role_arn = aws_stack.role_arn("sfn_role")
definition = clone(STATE_MACHINE_INTRINSIC_FUNCS)
lambda_arn_1 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_5)
lambda_arn_2 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_5)
if isinstance(definition["States"]["state1"].get("Parameters"), dict):
definition["States"]["state1"]["Parameters"]["lambda_params"][
"FunctionName"] = lambda_arn_1
definition["States"]["state3"]["Resource"] = lambda_arn_2
definition = json.dumps(definition)
sm_name = "intrinsic-%s" % short_uid()
self.sfn_client.create_state_machine(name=sm_name,
definition=definition,
roleArn=role_arn)
# run state machine
sm_arn = self.get_machine_arn(sm_name)
lambda_api.LAMBDA_EXECUTOR.function_invoke_times.clear()
input = {}
result = self.sfn_client.start_execution(stateMachineArn=sm_arn,
input=json.dumps(input))
self.assertTrue(result.get("executionArn"))
def check_invocations():
self.assertIn(lambda_arn_1,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
self.assertIn(lambda_arn_2,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual({"payload": {
"values": [1, "v2"]
}}, result.get("result_value"))
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=1, retries=10)
# clean up
self.cleanup(sm_arn, state_machines_before)
def test_create_run_state_machine(self):
state_machines_before = self.sfn_client.list_state_machines(
)['stateMachines']
# create state machine
role_arn = aws_stack.role_arn('sfn_role')
definition = clone(STATE_MACHINE_DEF)
lambda_arn_1 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_1)
lambda_arn_2 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_2)
definition['States']['step1']['Resource'] = lambda_arn_1
definition['States']['step2']['Resource'] = lambda_arn_2
definition = json.dumps(definition)
result = self.sfn_client.create_state_machine(name=STATE_MACHINE_NAME,
definition=definition,
roleArn=role_arn)
# assert that the SM has been created
self.assert_machine_created(state_machines_before)
# run state machine
state_machines = self.sfn_client.list_state_machines()['stateMachines']
sm_arn = [
m['stateMachineArn'] for m in state_machines
if m['name'] == STATE_MACHINE_NAME
][0]
result = self.sfn_client.start_execution(stateMachineArn=sm_arn)
self.assertTrue(result.get('executionArn'))
def check_invocations():
self.assertIn(lambda_arn_1,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
self.assertIn(lambda_arn_2,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual(result['result_value'],
{'Hello': TEST_RESULT_VALUE})
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=0.7, retries=25)
# clean up
self.sfn_client.delete_state_machine(stateMachineArn=sm_arn)
def test_intrinsic_functions(self):
state_machines_before = self.sfn_client.list_state_machines(
)['stateMachines']
# create state machine
role_arn = aws_stack.role_arn('sfn_role')
definition = clone(STATE_MACHINE_INTRINSIC_FUNCS)
lambda_arn_1 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_5)
lambda_arn_2 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_5)
if isinstance(definition['States']['state1'].get('Parameters'), dict):
definition['States']['state1']['Parameters']['lambda_params'][
'FunctionName'] = lambda_arn_1
definition['States']['state3']['Resource'] = lambda_arn_2
definition = json.dumps(definition)
sm_name = 'intrinsic-%s' % short_uid()
result = self.sfn_client.create_state_machine(name=sm_name,
definition=definition,
roleArn=role_arn)
# run state machine
sm_arn = self.get_machine_arn(sm_name)
lambda_api.LAMBDA_EXECUTOR.function_invoke_times.clear()
input = {}
result = self.sfn_client.start_execution(stateMachineArn=sm_arn,
input=json.dumps(input))
self.assertTrue(result.get('executionArn'))
def check_invocations():
self.assertIn(lambda_arn_1,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
self.assertIn(lambda_arn_2,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual({'payload': {
'values': [1, 'v2']
}}, result.get('result_value'))
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=1, retries=10)
# clean up
self.cleanup(sm_arn, state_machines_before)
def test_create_run_state_machine(self):
state_machines_before = self.sfn_client.list_state_machines(
)["stateMachines"]
# create state machine
role_arn = aws_stack.role_arn("sfn_role")
definition = clone(STATE_MACHINE_BASIC)
lambda_arn_1 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_1)
lambda_arn_2 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_2)
definition["States"]["step1"]["Resource"] = lambda_arn_1
definition["States"]["step2"]["Resource"] = lambda_arn_2
definition = json.dumps(definition)
sm_name = "basic-%s" % short_uid()
result = self.sfn_client.create_state_machine(name=sm_name,
definition=definition,
roleArn=role_arn)
# assert that the SM has been created
self.assert_machine_created(state_machines_before)
# run state machine
sm_arn = self.get_machine_arn(sm_name)
lambda_api.LAMBDA_EXECUTOR.function_invoke_times.clear()
result = self.sfn_client.start_execution(stateMachineArn=sm_arn)
self.assertTrue(result.get("executionArn"))
def check_invocations():
self.assertIn(lambda_arn_1,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
self.assertIn(lambda_arn_2,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual({"Hello": TEST_RESULT_VALUE},
result["result_value"])
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=0.7, retries=25)
# clean up
self.cleanup(sm_arn, state_machines_before)
def test_try_catch_state_machine(self):
state_machines_before = self.sfn_client.list_state_machines(
)['stateMachines']
# create state machine
role_arn = aws_stack.role_arn('sfn_role')
definition = clone(STATE_MACHINE_CATCH)
lambda_arn_1 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_1)
lambda_arn_2 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_2)
definition['States']['Start']['Parameters'][
'FunctionName'] = lambda_arn_1
definition['States']['ErrorHandler']['Resource'] = lambda_arn_2
definition['States']['Final']['Resource'] = lambda_arn_2
definition = json.dumps(definition)
sm_name = 'catch-%s' % short_uid()
result = self.sfn_client.create_state_machine(name=sm_name,
definition=definition,
roleArn=role_arn)
# run state machine
sm_arn = self.get_machine_arn(sm_name)
lambda_api.LAMBDA_EXECUTOR.function_invoke_times.clear()
result = self.sfn_client.start_execution(stateMachineArn=sm_arn)
self.assertTrue(result.get('executionArn'))
def check_invocations():
self.assertIn(lambda_arn_1,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
self.assertIn(lambda_arn_2,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual(result.get('handled'),
{'Hello': TEST_RESULT_VALUE})
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=1, retries=10)
# clean up
self.cleanup(sm_arn, state_machines_before)
def test_try_catch_state_machine(self, stepfunctions_client):
if os.environ.get("AWS_DEFAULT_REGION") != "us-east-1":
pytest.skip("skipping non us-east-1 temporarily")
state_machines_before = stepfunctions_client.list_state_machines(
)["stateMachines"]
# create state machine
role_arn = aws_stack.role_arn("sfn_role")
definition = clone(STATE_MACHINE_CATCH)
lambda_arn_1 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_1)
lambda_arn_2 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_2)
definition["States"]["Start"]["Parameters"][
"FunctionName"] = lambda_arn_1
definition["States"]["ErrorHandler"]["Resource"] = lambda_arn_2
definition["States"]["Final"]["Resource"] = lambda_arn_2
definition = json.dumps(definition)
sm_name = f"catch-{short_uid()}"
stepfunctions_client.create_state_machine(name=sm_name,
definition=definition,
roleArn=role_arn)
# run state machine
sm_arn = get_machine_arn(sm_name, stepfunctions_client)
lambda_api.LAMBDA_EXECUTOR.function_invoke_times.clear()
result = stepfunctions_client.start_execution(stateMachineArn=sm_arn)
assert result.get("executionArn")
def check_invocations():
assert lambda_arn_1 in lambda_api.LAMBDA_EXECUTOR.function_invoke_times
assert lambda_arn_2 in lambda_api.LAMBDA_EXECUTOR.function_invoke_times
# assert that the result is correct
result = _get_execution_results(sm_arn, stepfunctions_client)
assert {"Hello": TEST_RESULT_VALUE} == result.get("handled")
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=1, retries=10)
# clean up
cleanup(sm_arn, state_machines_before, stepfunctions_client)
def _test_api_gateway_lambda_proxy_integration(self, fn_name, path):
self.create_lambda_function(fn_name)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(fn_name)
invocation_uri = 'arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations'
target_uri = invocation_uri % (config.DEFAULT_REGION, lambda_uri)
result = self.connect_api_gateway_to_http_with_lambda_proxy(
'test_gateway2', target_uri, path=path)
api_id = result['id']
path_map = get_rest_api_paths(api_id)
_, resource = get_resource_for_path('/lambda/foo1', path_map)
# make test request to gateway and check response
path = path.replace('{test_param1}', 'foo1')
path = path + '?foo=foo&bar=bar&bar=baz'
url = self.gateway_request_url(
api_id=api_id, stage_name=self.TEST_STAGE_NAME, path=path)
data = {'return_status_code': 203, 'return_headers': {'foo': 'bar123'}}
result = requests.post(url, data=json.dumps(data),
headers={'User-Agent': 'python-requests/testing'})
self.assertEqual(result.status_code, 203)
self.assertEqual(result.headers.get('foo'), 'bar123')
self.assertIn('set-cookie', result.headers)
parsed_body = json.loads(to_str(result.content))
self.assertEqual(parsed_body.get('return_status_code'), 203)
self.assertDictEqual(parsed_body.get('return_headers'), {'foo': 'bar123'})
self.assertDictEqual(parsed_body.get('queryStringParameters'), {'foo': 'foo', 'bar': ['bar', 'baz']})
request_context = parsed_body.get('requestContext')
source_ip = request_context['identity'].pop('sourceIp')
self.assertTrue(re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', source_ip))
self.assertEqual(request_context['path'], '/lambda/foo1')
self.assertEqual(request_context['accountId'], TEST_AWS_ACCOUNT_ID)
self.assertEqual(request_context['resourceId'], resource.get('id'))
self.assertEqual(request_context['stage'], self.TEST_STAGE_NAME)
self.assertEqual(request_context['identity']['userAgent'], 'python-requests/testing')
result = requests.delete(url, data=json.dumps(data))
self.assertEqual(result.status_code, 204)
# send message with non-ASCII chars
body_msg = '🙀 - 参よ'
result = requests.post(url, data=json.dumps({'return_raw_body': body_msg}))
self.assertEqual(to_str(result.content), body_msg)
def test_publish_message_by_target_arn(self):
self.unsubscribe_all_from_sns()
topic_name = 'queue-{}'.format(short_uid())
func_name = 'lambda-%s' % short_uid()
topic_arn = self.sns_client.create_topic(Name=topic_name)['TopicArn']
testutil.create_lambda_function(handler_file=TEST_LAMBDA_ECHO_FILE,
func_name=func_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
lambda_arn = aws_stack.lambda_function_arn(func_name)
subscription_arn = self.sns_client.subscribe(
TopicArn=topic_arn, Protocol='lambda',
Endpoint=lambda_arn)['SubscriptionArn']
self.sns_client.publish(TopicArn=topic_arn,
Message='test_message_1',
Subject='test subject')
# Lambda invoked 1 time
events = retry(check_expected_lambda_log_events_length,
retries=3,
sleep=1,
function_name=func_name,
expected_length=1)
message = events[0]['Records'][0]
self.assertEqual(message['EventSubscriptionArn'], subscription_arn)
self.sns_client.publish(TargetArn=topic_arn,
Message='test_message_2',
Subject='test subject')
events = retry(check_expected_lambda_log_events_length,
retries=3,
sleep=1,
function_name=func_name,
expected_length=2)
# Lambda invoked 1 more time
self.assertEqual(len(events), 2)
for event in events:
message = event['Records'][0]
self.assertEqual(message['EventSubscriptionArn'], subscription_arn)
# clean up
self.sns_client.delete_topic(TopicArn=topic_arn)
lambda_client = aws_stack.connect_to_service('lambda')
lambda_client.delete_function(FunctionName=func_name)
def update_physical_resource_id(resource):
phys_res_id = getattr(resource, 'physical_resource_id', None)
if not phys_res_id:
if isinstance(resource, lambda_models.LambdaFunction):
func_arn = aws_stack.lambda_function_arn(resource.function_name)
resource.function_arn = resource.physical_resource_id = func_arn
elif isinstance(resource, sfn_models.StateMachine):
sm_arn = aws_stack.state_machine_arn(resource.name)
resource.physical_resource_id = sm_arn
elif isinstance(resource, service_models.StepFunctionsActivity):
act_arn = aws_stack.stepfunctions_activity_arn(resource.params.get('Name'))
resource.physical_resource_id = act_arn
else:
LOG.warning('Unable to determine physical_resource_id for resource %s' % type(resource))
def update_physical_resource_id(resource):
phys_res_id = getattr(resource, "physical_resource_id", None)
if phys_res_id:
return
if isinstance(resource, lambda_models.LambdaFunction):
func_arn = aws_stack.lambda_function_arn(resource.function_name)
resource.function_arn = resource.physical_resource_id = func_arn
elif isinstance(resource, service_models.StepFunctionsActivity):
act_arn = aws_stack.stepfunctions_activity_arn(
resource.params.get("Name"))
resource.physical_resource_id = act_arn
elif isinstance(resource, kinesis_models.Stream):
resource.physical_resource_id = resource.stream_name
elif isinstance(resource, logs.LogsLogGroup):
resource.physical_resource_id = resource.params.get("LogGroupName")
elif isinstance(resource, kinesisfirehose.FirehoseDeliveryStream):
resource.physical_resource_id = resource.params.get(
"DeliveryStreamName")
elif isinstance(resource, events.EventsRule):
resource.physical_resource_id = resource.params.get("Name")
elif isinstance(resource, elasticsearch.ElasticsearchDomain):
resource.physical_resource_id = resource.params.get("DomainName")
elif isinstance(resource, dynamodb_models.Table):
resource.physical_resource_id = resource.name
elif isinstance(resource, dynamodb2_models.Table):
resource.physical_resource_id = resource.name
elif isinstance(resource, apigw_models.RestAPI):
resource.physical_resource_id = resource.id
elif isinstance(resource, apigw_models.Stage):
resource.physical_resource_id = resource.get("stageName")
elif isinstance(resource, apigw_models.Resource):
resource.physical_resource_id = resource.id
else:
LOG.warning(
"Unable to determine physical_resource_id for resource %s" %
type(resource))
def update_physical_resource_id(resource):
phys_res_id = getattr(resource, 'physical_resource_id') if hasattr(
resource, 'physical_resource_id') else None
if not phys_res_id:
if isinstance(resource, lambda_models.LambdaFunction):
func_arn = aws_stack.lambda_function_arn(
resource.function_name)
resource.function_arn = resource.physical_resource_id = func_arn
elif isinstance(resource, sfn_models.StateMachine):
sm_arn = aws_stack.state_machine_arn(resource.name)
resource.physical_resource_id = sm_arn
else:
LOG.warning(
'Unable to determine physical_resource_id for resource %s'
% type(resource))
def fetch_state(self, stack_name, resources):
props = self.props
resource_id = props['FunctionName'] or self.resource_id
source_arn = props.get('EventSourceArn')
resource_id = self.resolve_refs_recursively(stack_name, resource_id, resources)
source_arn = self.resolve_refs_recursively(stack_name, source_arn, resources)
if not resource_id or not source_arn:
raise Exception('ResourceNotFound')
mappings = aws_stack.connect_to_service('lambda').list_event_source_mappings(
FunctionName=resource_id, EventSourceArn=source_arn)
mapping = list(filter(lambda m:
m['EventSourceArn'] == source_arn and m['FunctionArn'] == aws_stack.lambda_function_arn(resource_id),
mappings['EventSourceMappings']))
if not mapping:
raise Exception('ResourceNotFound')
return mapping[0]
def fetch_state(self, stack_name, resources):
iam = aws_stack.connect_to_service('iam')
props = self.props
policy_name = LAMBDA_POLICY_NAME_PATTERN % props.get('FunctionName')
policy_arn = aws_stack.policy_arn(policy_name)
policy = iam.get_policy(PolicyArn=policy_arn)['Policy']
version = policy.get('DefaultVersionId')
policy = iam.get_policy_version(PolicyArn=policy_arn, VersionId=version)['PolicyVersion']
statements = policy['Document']['Statement']
statements = statements if isinstance(statements, list) else [statements]
func_arn = aws_stack.lambda_function_arn(props['FunctionName'])
principal = props.get('Principal')
existing = [s for s in statements if s['Action'] == props['Action'] and
s['Resource'] == func_arn and
(not principal or s['Principal'] in [{'Service': principal}, {'Service': [principal]}])]
return existing[0] if existing else None
def create_lambda_api_gateway_integration(gateway_name, func_name, handler_file,
methods=[], path=None, runtime=None, stage_name=None, auth_type=None):
methods = methods or ['GET', 'POST']
path = path or '/test'
auth_type = auth_type or 'REQUEST'
stage_name = stage_name or 'test'
# create Lambda
zip_file = create_lambda_archive(handler_file, get_content=True, runtime=runtime)
create_lambda_function(func_name=func_name, zip_file=zip_file, runtime=runtime)
func_arn = aws_stack.lambda_function_arn(func_name)
target_arn = aws_stack.apigateway_invocations_arn(func_arn)
# connect API GW to Lambda
result = connect_api_gateway_to_http_with_lambda_proxy(
gateway_name, target_arn, stage_name=stage_name, path=path, auth_type=auth_type)
return result
def test_http_invocation_with_apigw_proxy(self):
lambda_name = "test_lambda_%s" % short_uid()
lambda_resource = "/api/v1/{proxy+}"
lambda_path = "/api/v1/hello/world"
lambda_request_context_path = "/" + TEST_STAGE_NAME + lambda_path
lambda_request_context_resource_path = lambda_resource
# create lambda function
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
func_name=lambda_name,
)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(lambda_name)
invocation_uri = "arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations"
target_uri = invocation_uri % (aws_stack.get_region(), lambda_uri)
result = testutil.connect_api_gateway_to_http_with_lambda_proxy(
"test_gateway2",
target_uri,
path=lambda_resource,
stage_name=TEST_STAGE_NAME,
)
api_id = result["id"]
url = gateway_request_url(api_id=api_id,
stage_name=TEST_STAGE_NAME,
path=lambda_path)
result = safe_requests.post(
url, data=b"{}", headers={"User-Agent": "python-requests/testing"})
content = json.loads(result.content)
self.assertEqual(lambda_path, content["path"])
self.assertEqual(lambda_resource, content["resource"])
self.assertEqual(lambda_request_context_path,
content["requestContext"]["path"])
self.assertEqual(
lambda_request_context_resource_path,
content["requestContext"]["resourcePath"],
)
# clean up
testutil.delete_lambda_function(lambda_name)
def test_create_run_map_state_machine(self):
names = ['Bob', 'Meg', 'Joe']
test_input = [{'map': name} for name in names]
test_output = [{'Hello': name} for name in names]
state_machines_before = self.sfn_client.list_state_machines(
)['stateMachines']
role_arn = aws_stack.role_arn('sfn_role')
definition = clone(MAP_STATE_MACHINE_DEF)
lambda_arn_3 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_3)
definition['States']['ExampleMapState']['Iterator']['States'][
'CallLambda']['Resource'] = lambda_arn_3
definition = json.dumps(definition)
result = self.sfn_client.create_state_machine(
name=MAP_STATE_MACHINE_NAME,
definition=definition,
roleArn=role_arn)
# assert that the SM has been created
state_machines_after = self.sfn_client.list_state_machines(
)['stateMachines']
self.assertEqual(len(state_machines_after),
len(state_machines_before) + 1)
# run state machine
sm_arn = [
m['stateMachineArn'] for m in state_machines_after
if m['name'] == MAP_STATE_MACHINE_NAME
][0]
result = self.sfn_client.start_execution(stateMachineArn=sm_arn,
input=json.dumps(test_input))
self.assertTrue(result.get('executionArn'))
def check_invocations():
self.assertIn(lambda_arn_3,
lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual(result, test_output)
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=1, retries=10)
# clean up
self.sfn_client.delete_state_machine(stateMachineArn=sm_arn)
def test_redrive_policy_lambda_subscription(self):
self.unsubscribe_all_from_sns()
lambda_name = "test-%s" % short_uid()
lambda_arn = aws_stack.lambda_function_arn(lambda_name)
testutil.create_lambda_function(
func_name=lambda_name,
libs=TEST_LAMBDA_LIBS,
handler_file=TEST_LAMBDA_PYTHON,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
subscription = self.sns_client.subscribe(TopicArn=self.topic_arn,
Protocol="lambda",
Endpoint=lambda_arn)
self.sns_client.set_subscription_attributes(
SubscriptionArn=subscription["SubscriptionArn"],
AttributeName="RedrivePolicy",
AttributeValue=json.dumps({
"deadLetterTargetArn":
aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME)
}),
)
testutil.delete_lambda_function(lambda_name)
self.sns_client.publish(
TopicArn=self.topic_arn,
Message=json.dumps({"message": "test_redrive_policy"}),
)
def receive_dlq():
result = self.sqs_client.receive_message(
QueueUrl=self.dlq_url, MessageAttributeNames=["All"])
self.assertGreater(len(result["Messages"]), 0)
self.assertEqual(
json.loads(
json.loads(result["Messages"][0]["Body"])["Message"][0])
["message"],
"test_redrive_policy",
)
retry(receive_dlq, retries=10, sleep=2)
def update_physical_resource_id(resource):
phys_res_id = getattr(resource, 'physical_resource_id', None)
if not phys_res_id:
if isinstance(resource, lambda_models.LambdaFunction):
func_arn = aws_stack.lambda_function_arn(resource.function_name)
resource.function_arn = resource.physical_resource_id = func_arn
elif isinstance(resource, sfn_models.StateMachine):
sm_arn = aws_stack.state_machine_arn(resource.name)
resource.physical_resource_id = sm_arn
elif isinstance(resource, service_models.StepFunctionsActivity):
act_arn = aws_stack.stepfunctions_activity_arn(resource.params.get('Name'))
resource.physical_resource_id = act_arn
elif isinstance(resource, kinesis_models.Stream):
resource.physical_resource_id = resource.stream_name
elif isinstance(resource, service_models.LogsLogGroup):
resource.physical_resource_id = resource.params.get('LogGroupName')
elif isinstance(resource, service_models.FirehoseDeliveryStream):
resource.physical_resource_id = resource.params.get('DeliveryStreamName')
elif isinstance(resource, service_models.SecretsManagerSecret):
resource.physical_resource_id = resource.params.get('Name')
elif isinstance(resource, service_models.EventsRule):
resource.physical_resource_id = resource.params.get('Name')
elif isinstance(resource, service_models.ElasticsearchDomain):
resource.physical_resource_id = resource.params.get('DomainName')
elif isinstance(resource, dynamodb_models.Table):
resource.physical_resource_id = resource.name
elif isinstance(resource, dynamodb2_models.Table):
resource.physical_resource_id = resource.name
elif isinstance(resource, apigw_models.RestAPI):
resource.physical_resource_id = resource.id
else:
LOG.warning('Unable to determine physical_resource_id for resource %s' % type(resource))
def test_redrive_policy_lambda_subscription(self):
self.unsubscripe_all_from_sns()
lambda_name = 'test-%s' % short_uid()
lambda_arn = aws_stack.lambda_function_arn(lambda_name)
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
testutil.create_lambda_function(func_name=lambda_name,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON36)
subscription = self.sns_client.subscribe(TopicArn=self.topic_arn,
Protocol='lambda',
Endpoint=lambda_arn)
self.sns_client.set_subscription_attributes(
SubscriptionArn=subscription['SubscriptionArn'],
AttributeName='RedrivePolicy',
AttributeValue=json.dumps({
'deadLetterTargetArn':
aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME)
}))
testutil.delete_lambda_function(lambda_name)
self.sns_client.publish(TopicArn=self.topic_arn,
Message=json.dumps(
{'message': 'test_redrive_policy'}))
def receive_dlq():
result = self.sqs_client.receive_message(
QueueUrl=self.dlq_url, MessageAttributeNames=['All'])
self.assertGreater(len(result['Messages']), 0)
self.assertEqual(
json.loads(
json.loads(result['Messages'][0]['Body'])['Message'][0])
['message'], 'test_redrive_policy')
retry(receive_dlq, retries=10, sleep=2)
def test_create_choice_state_machine(self):
state_machines_before = self.sfn_client.list_state_machines(
)['stateMachines']
role_arn = aws_stack.role_arn('sfn_role')
definition = clone(CHOICE_STATE_MACHINE_DEF)
lambda_arn_4 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_4)
definition['States']['Add']['Resource'] = lambda_arn_4
definition = json.dumps(definition)
result = self.sfn_client.create_state_machine(
name=CHOICE_STATE_MACHINE_NAME,
definition=definition,
roleArn=role_arn)
# assert that the SM has been created
state_machines_after = self.sfn_client.list_state_machines(
)['stateMachines']
self.assertEqual(len(state_machines_after),
len(state_machines_before) + 1)
# run state machine
state_machines = self.sfn_client.list_state_machines()['stateMachines']
sm_arn = [
m['stateMachineArn'] for m in state_machines
if m['name'] == CHOICE_STATE_MACHINE_NAME
][0]
input = {'x': '1', 'y': '2'}
result = self.sfn_client.start_execution(stateMachineArn=sm_arn,
input=json.dumps(input))
self.assertTrue(result.get('executionArn'))
# define expected output
test_output = {**input, 'added': {'Hello': TEST_RESULT_VALUE}}
def check_result():
result = self._get_execution_results(sm_arn)
self.assertEqual(result, test_output)
# assert that the result is correct
retry(check_result, sleep=2, retries=10)
# clean up
self.sfn_client.delete_state_machine(stateMachineArn=sm_arn)
def test_dead_letter_queue(self):
lambda_name = 'test-%s' % short_uid()
lambda_arn = aws_stack.lambda_function_arn(lambda_name)
topic_name = 'test-%s' % short_uid()
topic_arn = self.sns_client.create_topic(Name=topic_name)['TopicArn']
queue_name = 'test-%s' % short_uid()
queue_url = self.sqs_client.create_queue(
QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
testutil.create_lambda_function(
func_name=lambda_name,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON36,
DeadLetterConfig={'TargetArn': queue_arn},
)
self.sns_client.subscribe(TopicArn=topic_arn,
Protocol='lambda',
Endpoint=lambda_arn)
payload = {
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG: 1,
}
self.sns_client.publish(TopicArn=topic_arn,
Message=json.dumps(payload))
def receive_dlq():
result = self.sqs_client.receive_message(
QueueUrl=queue_url, MessageAttributeNames=['All'])
msg_attrs = result['Messages'][0]['MessageAttributes']
self.assertGreater(len(result['Messages']), 0)
self.assertIn('RequestID', msg_attrs)
self.assertIn('ErrorCode', msg_attrs)
self.assertIn('ErrorMessage', msg_attrs)
retry(receive_dlq, retries=8, sleep=2)
def test_create_run_map_state_machine(self):
names = ["Bob", "Meg", "Joe"]
test_input = [{"map": name} for name in names]
test_output = [{"Hello": name} for name in names]
state_machines_before = self.sfn_client.list_state_machines()["stateMachines"]
role_arn = aws_stack.role_arn("sfn_role")
definition = clone(STATE_MACHINE_MAP)
lambda_arn_3 = aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_3)
definition["States"]["ExampleMapState"]["Iterator"]["States"]["CallLambda"][
"Resource"
] = lambda_arn_3
definition = json.dumps(definition)
sm_name = "map-%s" % short_uid()
_ = self.sfn_client.create_state_machine(
name=sm_name, definition=definition, roleArn=role_arn
)
# assert that the SM has been created
self.assert_machine_created(state_machines_before)
# run state machine
sm_arn = self.get_machine_arn(sm_name)
lambda_api.LAMBDA_EXECUTOR.function_invoke_times.clear()
result = self.sfn_client.start_execution(
stateMachineArn=sm_arn, input=json.dumps(test_input)
)
self.assertTrue(result.get("executionArn"))
def check_invocations():
self.assertIn(lambda_arn_3, lambda_api.LAMBDA_EXECUTOR.function_invoke_times)
# assert that the result is correct
result = self._get_execution_results(sm_arn)
self.assertEqual(test_output, result)
# assert that the lambda has been invoked by the SM execution
retry(check_invocations, sleep=1, retries=10)
# clean up
self.cleanup(sm_arn, state_machines_before)
def _test_api_gateway_lambda_proxy_integration_any_method(self, fn_name, path):
# create lambda function
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27
)
testutil.create_lambda_function(
func_name=fn_name,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON27
)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(fn_name)
target_uri = aws_stack.apigateway_invocations_arn(lambda_uri)
result = self.connect_api_gateway_to_http_with_lambda_proxy(
'test_gateway3',
target_uri,
methods=['ANY'],
path=path
)
# make test request to gateway and check response
path = path.replace('{test_param1}', 'foo1')
url = INBOUND_GATEWAY_URL_PATTERN.format(
api_id=result['id'],
stage_name=self.TEST_STAGE_NAME,
path=path
)
data = {}
for method in ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'):
body = json.dumps(data) if method in ('POST', 'PUT', 'PATCH') else None
result = getattr(requests, method.lower())(url, data=body)
self.assertEqual(result.status_code, 200)
parsed_body = json.loads(to_str(result.content))
self.assertEqual(parsed_body.get('httpMethod'), method)
def _test_api_gateway_lambda_proxy_integration_any_method(self, fn_name, path):
self.create_lambda_function(fn_name)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(fn_name)
target_uri = aws_stack.apigateway_invocations_arn(lambda_uri)
result = self.connect_api_gateway_to_http_with_lambda_proxy(
'test_gateway3', target_uri, methods=['ANY'], path=path)
# make test request to gateway and check response
path = path.replace('{test_param1}', 'foo1')
url = self.gateway_request_url(
api_id=result['id'], stage_name=self.TEST_STAGE_NAME, path=path)
data = {}
for method in ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'):
body = json.dumps(data) if method in ('POST', 'PUT', 'PATCH') else None
result = getattr(requests, method.lower())(url, data=body)
self.assertEqual(result.status_code, 200)
parsed_body = json.loads(to_str(result.content))
self.assertEqual(parsed_body.get('httpMethod'), method)
def test_apigateway_with_custom_authorization_method(self):
apigw_client = aws_stack.connect_to_service('apigateway')
# create Lambda function
lambda_name = 'apigw-lambda-%s' % short_uid()
self.create_lambda_function(lambda_name)
lambda_uri = aws_stack.lambda_function_arn(lambda_name)
# create REST API
api = apigw_client.create_rest_api(name='test-api', description='')
api_id = api['id']
root_res_id = apigw_client.get_resources(
restApiId=api_id)['items'][0]['id']
# create authorizer at root resource
authorizer = apigw_client.create_authorizer(
restApiId=api_id,
name='lambda_authorizer',
type='TOKEN',
authorizerUri='arn:aws:apigateway:us-east-1:lambda:path/ \
2015-03-31/functions/{}/invocations'.format(lambda_uri),
identitySource='method.request.header.Auth')
# create method with custom authorizer
is_api_key_required = True
method_response = apigw_client.put_method(
restApiId=api_id,
resourceId=root_res_id,
httpMethod='GET',
authorizationType='CUSTOM',
authorizerId=authorizer['id'],
apiKeyRequired=is_api_key_required)
self.assertEqual(authorizer['id'], method_response['authorizerId'])
# clean up
lambda_client = aws_stack.connect_to_service('lambda')
lambda_client.delete_function(FunctionName=lambda_name)
apigw_client.delete_rest_api(restApiId=api_id)
def test_lambda_aws_integration(apigateway_client):
fn_name = f"test-{short_uid()}"
create_lambda_function(
func_name=fn_name,
handler_file=TEST_LAMBDA_HELLO_WORLD,
handler="lambda_hello_world.handler",
runtime=LAMBDA_RUNTIME_PYTHON39,
)
lambda_arn = aws_stack.lambda_function_arn(fn_name)
api_id, _, root = create_rest_api(apigateway_client, name="aws lambda api")
resource_id, _ = create_rest_resource(apigateway_client,
restApiId=api_id,
parentId=root,
pathPart="test")
# create method and integration
create_rest_resource_method(
apigateway_client,
restApiId=api_id,
resourceId=resource_id,
httpMethod="GET",
authorizationType="NONE",
)
create_rest_api_integration(
apigateway_client,
restApiId=api_id,
resourceId=resource_id,
httpMethod="GET",
integrationHttpMethod="GET",
type="AWS",
uri=
f"arn:aws:apigateway:{aws_stack.get_region()}:lambda:path//2015-03-31/functions/{lambda_arn}/invocations",
)
url = api_invoke_url(api_id=api_id, stage="local", path="/test")
response = requests.get(url)
assert response.json() == {"message": "Hello from Lambda"}
def create_lambda_api_gateway_integration(
gateway_name,
func_name,
handler_file,
methods=None,
path=None,
runtime=None,
stage_name=None,
auth_type=None,
auth_creator_func=None,
):
if methods is None:
methods = []
path = path or "/test"
auth_type = auth_type or "REQUEST"
stage_name = stage_name or "test"
# create Lambda
zip_file = create_lambda_archive(handler_file,
get_content=True,
runtime=runtime)
create_lambda_function(func_name=func_name,
zip_file=zip_file,
runtime=runtime)
func_arn = aws_stack.lambda_function_arn(func_name)
target_arn = aws_stack.apigateway_invocations_arn(func_arn)
# connect API GW to Lambda
result = connect_api_gateway_to_http_with_lambda_proxy(
gateway_name,
target_arn,
stage_name=stage_name,
path=path,
methods=methods,
auth_type=auth_type,
auth_creator_func=auth_creator_func,
)
return result
def test_http_invocation_with_apigw_proxy(self, create_lambda_function):
lambda_name = f"test_lambda_{short_uid()}"
lambda_resource = "/api/v1/{proxy+}"
lambda_path = "/api/v1/hello/world"
lambda_request_context_path = "/" + TEST_STAGE_NAME + lambda_path
lambda_request_context_resource_path = lambda_resource
# create lambda function
create_lambda_function(
func_name=lambda_name,
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(lambda_name)
target_uri = f"arn:aws:apigateway:{aws_stack.get_region()}:lambda:path/2015-03-31/functions/{lambda_uri}/invocations"
result = testutil.connect_api_gateway_to_http_with_lambda_proxy(
"test_gateway2",
target_uri,
path=lambda_resource,
stage_name=TEST_STAGE_NAME,
)
api_id = result["id"]
url = path_based_url(api_id=api_id,
stage_name=TEST_STAGE_NAME,
path=lambda_path)
result = safe_requests.post(
url, data=b"{}", headers={"User-Agent": "python-requests/testing"})
content = json.loads(result.content)
assert lambda_path == content["path"]
assert lambda_resource == content["resource"]
assert lambda_request_context_path == content["requestContext"]["path"]
assert lambda_request_context_resource_path == content[
"requestContext"]["resourcePath"]
def test_apigateway_deployed(self):
function_name = 'sls-test-local-router'
lambda_client = aws_stack.connect_to_service('lambda')
resp = lambda_client.list_functions()
function = [fn for fn in resp['Functions'] if fn['FunctionName'] == function_name][0]
self.assertEqual(function['Handler'], 'src/http.router')
apigw_client = aws_stack.connect_to_service('apigateway')
apis = apigw_client.get_rest_apis()['items']
api_ids = [api['id'] for api in apis if api['id'] not in self.api_ids]
self.assertEqual(len(api_ids), 1)
resources = apigw_client.get_resources(restApiId=api_ids[0])['items']
proxy_resources = [res for res in resources if res['path'] == '/{proxy+}']
self.assertEqual(len(proxy_resources), 1)
proxy_resource = proxy_resources[0]
for method in ['DELETE', 'OPTIONS', 'GET', 'POST', 'PUT', 'PATCH', 'HEAD']:
self.assertIn(method, proxy_resource['resourceMethods'])
resource_method = proxy_resource['resourceMethods'][method]
self.assertIn(aws_stack.lambda_function_arn(function_name), resource_method['methodIntegration']['uri'])
def fetch_state(self, stack_name, resources):
props = self.props
source_arn = props.get("EventSourceArn")
self_managed_src = props.get("SelfManagedEventSource")
function_name = self.resolve_refs_recursively(stack_name, props["FunctionName"], resources)
source_arn = self.resolve_refs_recursively(stack_name, source_arn, resources)
if not function_name or (not source_arn and not self_managed_src):
raise Exception("ResourceNotFound")
def _matches(m):
return m["FunctionArn"] == lambda_arn and (
m.get("EventSourceArn") == source_arn
or m.get("SelfManagedEventSource") == self_managed_src
)
client = aws_stack.connect_to_service("lambda")
lambda_arn = aws_stack.lambda_function_arn(function_name)
kwargs = {"EventSourceArn": source_arn} if source_arn else {}
mappings = client.list_event_source_mappings(FunctionName=function_name, **kwargs)
mapping = list(filter(lambda m: _matches(m), mappings["EventSourceMappings"]))
if not mapping:
raise Exception("ResourceNotFound")
return mapping[0]
def test_api_gateway_lambda_proxy_integration():
# create lambda function
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_PROXY_BACKEND,
zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(TEST_LAMBDA_PROXY_BACKEND)
target_uri = 'arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations' % (DEFAULT_REGION, lambda_uri)
result = connect_api_gateway_to_http_with_lambda_proxy('test_gateway2', target_uri,
path=API_PATH_LAMBDA_PROXY_BACKEND)
# make test request to gateway and check response
path = API_PATH_LAMBDA_PROXY_BACKEND.replace('{test_param1}', 'foo1')
url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'], stage_name=TEST_STAGE_NAME, path=path)
data = {'return_status_code': 203, 'return_headers': {'foo': 'bar123'}}
result = requests.post(url, data=json.dumps(data))
assert result.status_code == 203
assert result.headers.get('foo') == 'bar123'
parsed_body = json.loads(to_str(result.content))
assert parsed_body.get('return_status_code') == 203
assert parsed_body.get('return_headers') == {'foo': 'bar123'}
assert parsed_body.get('pathParameters') == {'test_param1': 'foo1'}
def func_arn(function_name):
return aws_stack.lambda_function_arn(function_name)
def test_kinesis_lambda_sns_ddb_streams():
ddb_lease_table_suffix = '-kclapp'
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(
TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False)
aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
EVENTS.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records,
wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == TEST_TABLE_NAME:
ddb_event_source_arn = stream['StreamArn']
assert ddb_event_source_arn
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN']
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(TEST_TABLE_NAME)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}},
{'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}}
]})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(TableName=TEST_TABLE_NAME,
Key={PARTITION_KEY: {'S': 'testId%s' % i}},
AttributeUpdates={'data': {
'Action': 'PUT',
'Value': {'S': 'foobar123_updated'}
}})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(
Records=[
{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)
], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test message
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000', count=10)
assert len(latest) == 10
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events = num_events_ddb + num_events_kinesis + num_events_sns
def check_events():
if len(EVENTS) != num_events:
LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' +
'(actual/expected): %s/%s') % (len(EVENTS), num_events))
assert len(EVENTS) == num_events
event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS]
inserts = [e for e in event_items if e.get('__action_type') == 'INSERT']
modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY']
assert len(inserts) == num_put_new_items + num_batch_items
assert len(modifies) == num_put_existing_items + num_updates_ddb
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=7, sleep=3)
# make sure the we have the right amount of INSERT/MODIFY event types
# check cloudwatch notifications
stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM)
assert len(stats1['Datapoints']) == 2 + num_events_sns
stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors')
assert len(stats2['Datapoints']) == 1
stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB)
assert len(stats3['Datapoints']) == num_events_ddb
def retrieve_resource_details(resource_id, resource_status, resources, stack_name):
resource = resources[resource_id]
resource_id = resource_status.get('PhysicalResourceId') or resource_id
resource_type = resource_status['ResourceType']
if not resource:
resource = {}
resource_props = resource.get('Properties')
try:
if resource_type == 'AWS::Lambda::Function':
resource_id = resource_props['FunctionName'] if resource else resource_id
return aws_stack.connect_to_service('lambda').get_function(FunctionName=resource_id)
if resource_type == 'AWS::Lambda::EventSourceMapping':
resource_id = resource_props['FunctionName'] if resource else resource_id
source_arn = resource_props.get('EventSourceArn')
resource_id = resolve_refs_recursively(stack_name, resource_id, resources)
source_arn = resolve_refs_recursively(stack_name, source_arn, resources)
if not resource_id or not source_arn:
raise Exception('ResourceNotFound')
mappings = aws_stack.connect_to_service('lambda').list_event_source_mappings(
FunctionName=resource_id, EventSourceArn=source_arn)
mapping = list(filter(lambda m:
m['EventSourceArn'] == source_arn and m['FunctionArn'] == aws_stack.lambda_function_arn(resource_id),
mappings['EventSourceMappings']))
if not mapping:
raise Exception('ResourceNotFound')
return mapping[0]
if resource_type == 'AWS::DynamoDB::Table':
resource_id = resource_props['TableName'] if resource else resource_id
return aws_stack.connect_to_service('dynamodb').describe_table(TableName=resource_id)
if resource_type == 'AWS::ApiGateway::RestApi':
apis = aws_stack.connect_to_service('apigateway').get_rest_apis()['items']
resource_id = resource_props['Name'] if resource else resource_id
result = list(filter(lambda api: api['name'] == resource_id, apis))
return result[0] if result else None
if resource_type == 'AWS::ApiGateway::Resource':
api_id = resource_props['RestApiId'] if resource else resource_id
api_id = resolve_refs_recursively(stack_name, api_id, resources)
parent_id = resolve_refs_recursively(stack_name, resource_props['ParentId'], resources)
if not api_id or not parent_id:
return None
api_resources = aws_stack.connect_to_service('apigateway').get_resources(restApiId=api_id)['items']
target_resource = list(filter(lambda res:
res.get('parentId') == parent_id and res['pathPart'] == resource_props['PathPart'], api_resources))
if not target_resource:
return None
path = aws_stack.get_apigateway_path_for_resource(api_id,
target_resource[0]['id'], resources=api_resources)
result = list(filter(lambda res: res['path'] == path, api_resources))
return result[0] if result else None
if resource_type == 'AWS::ApiGateway::Deployment':
api_id = resource_props['RestApiId'] if resource else resource_id
api_id = resolve_refs_recursively(stack_name, api_id, resources)
if not api_id:
return None
result = aws_stack.connect_to_service('apigateway').get_deployments(restApiId=api_id)['items']
# TODO possibly filter results by stage name or other criteria
return result[0] if result else None
if resource_type == 'AWS::ApiGateway::Method':
api_id = resolve_refs_recursively(stack_name, resource_props['RestApiId'], resources)
res_id = resolve_refs_recursively(stack_name, resource_props['ResourceId'], resources)
if not api_id or not res_id:
return None
return aws_stack.connect_to_service('apigateway').get_method(restApiId=api_id,
resourceId=res_id, httpMethod=resource_props['HttpMethod'])
if resource_type == 'AWS::SQS::Queue':
queues = aws_stack.connect_to_service('sqs').list_queues()
result = list(filter(lambda item:
# TODO possibly find a better way to compare resource_id with queue URLs
item.endswith('/%s' % resource_id), queues.get('QueueUrls', [])))
return result
if resource_type == 'AWS::S3::Bucket':
return aws_stack.connect_to_service('s3').get_bucket_location(Bucket=resource_id)
if resource_type == 'AWS::Logs::LogGroup':
# TODO implement
raise Exception('ResourceNotFound')
if resource_type == 'AWS::Kinesis::Stream':
stream_name = resolve_refs_recursively(stack_name, resource_props['Name'], resources)
result = aws_stack.connect_to_service('kinesis').describe_stream(StreamName=stream_name)
return result
if is_deployable_resource(resource):
LOGGER.warning('Unexpected resource type %s when resolving references' % resource_type)
except Exception as e:
# we expect this to be a "not found" exception
markers = ['NoSuchBucket', 'ResourceNotFound', '404']
if not list(filter(lambda marker, e=e: marker in str(e), markers)):
LOGGER.warning('Unexpected error retrieving details for resource %s: %s %s - %s %s' %
(resource_type, e, traceback.format_exc(), resource, resource_status))
return None
