def test_python_lambda_running_in_docker(self):
if not use_docker():
return
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON3),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON36
)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_PY3,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON36
)
result = self.lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertEqual(to_str(result_data).strip(), '{}')
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_PY3)
def test_java_runtime_with_lib(self):
java_jar_with_lib = load_file(TEST_LAMBDA_JAVA_WITH_LIB, mode='rb')
# create ZIP file from JAR file
jar_dir = new_tmp_dir()
zip_dir = new_tmp_dir()
unzip(TEST_LAMBDA_JAVA_WITH_LIB, jar_dir)
shutil.move(os.path.join(jar_dir, 'lib'), os.path.join(zip_dir, 'lib'))
jar_without_libs_file = testutil.create_zip_file(jar_dir)
shutil.copy(jar_without_libs_file,
os.path.join(zip_dir, 'lib', 'lambda.jar'))
java_zip_with_lib = testutil.create_zip_file(zip_dir, get_content=True)
for archive in [java_jar_with_lib, java_zip_with_lib]:
lambda_name = 'test-%s' % short_uid()
testutil.create_lambda_function(
func_name=lambda_name,
zip_file=archive,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.LambdaHandlerWithLib')
result = self.lambda_client.invoke(FunctionName=lambda_name,
Payload=b'{"echo":"echo"}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertIn('echo', to_str(result_data))
# clean up
testutil.delete_lambda_function(lambda_name)
def test_nodejs_lambda_running_in_docker(self):
if not use_docker():
return
zip_file = testutil.create_zip_file(
TEST_LAMBDA_NODEJS, get_content=True)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JS,
zip_file=zip_file,
handler='lambda_integration.handler',
runtime=LAMBDA_RUNTIME_NODEJS810
)
result = self.lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertEqual(to_str(result_data).strip(), '{}')
# assert that logs are present
expected = ['.*Node.js Lambda handler executing.']
self.check_lambda_logs(TEST_LAMBDA_NAME_JS, expected_lines=expected)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_JS)
def test_put_events_with_target_lambda(self):
rule_name = "rule-{}".format(short_uid())
function_name = "lambda-func-{}".format(short_uid())
target_id = "target-{}".format(short_uid())
bus_name = "bus-{}".format(short_uid())
rs = testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON_ECHO,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
func_arn = rs["CreateFunctionResponse"]["FunctionArn"]
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN),
)
rs = self.events_client.put_targets(
Rule=rule_name,
EventBusName=bus_name,
Targets=[{
"Id": target_id,
"Arn": func_arn
}],
)
self.assertIn("FailedEntryCount", rs)
self.assertIn("FailedEntries", rs)
self.assertEqual(0, rs["FailedEntryCount"])
self.assertEqual([], rs["FailedEntries"])
self.events_client.put_events(
Entries=[{
"EventBusName": bus_name,
"Source": TEST_EVENT_PATTERN["Source"][0],
"DetailType": TEST_EVENT_PATTERN["detail-type"][0],
"Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]),
}])
# Get lambda's log events
events = retry(
check_expected_lambda_log_events_length,
retries=3,
sleep=1,
function_name=function_name,
expected_length=1,
)
actual_event = events[0]
self.assertIsValidEvent(actual_event)
self.assertDictEqual(
json.loads(actual_event["detail"]),
json.loads(TEST_EVENT_PATTERN["Detail"][0]),
)
# clean up
testutil.delete_lambda_function(function_name)
self.cleanup(bus_name, rule_name, target_id)
def test_redrive_policy_lambda_subscription(self):
self.unsubscribe_all_from_sns()
lambda_name = 'test-%s' % short_uid()
lambda_arn = aws_stack.lambda_function_arn(lambda_name)
testutil.create_lambda_function(func_name=lambda_name, libs=TEST_LAMBDA_LIBS,
handler_file=TEST_LAMBDA_PYTHON, runtime=LAMBDA_RUNTIME_PYTHON36)
subscription = self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol='lambda', Endpoint=lambda_arn)
self.sns_client.set_subscription_attributes(
SubscriptionArn=subscription['SubscriptionArn'],
AttributeName='RedrivePolicy',
AttributeValue=json.dumps({'deadLetterTargetArn': aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME)})
)
testutil.delete_lambda_function(lambda_name)
self.sns_client.publish(TopicArn=self.topic_arn, Message=json.dumps({'message': 'test_redrive_policy'}))
def receive_dlq():
result = self.sqs_client.receive_message(QueueUrl=self.dlq_url, MessageAttributeNames=['All'])
self.assertGreater(len(result['Messages']), 0)
self.assertEqual(
json.loads(json.loads(result['Messages'][0]['Body'])['Message'][0])['message'],
'test_redrive_policy'
)
retry(receive_dlq, retries=10, sleep=2)
def test_lambda_environment(self):
vars = {'Hello': 'World'}
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_ENV),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_ENV,
zip_file=zip_file,
runtime=LAMBDA_RUNTIME_PYTHON27,
envvars=vars)
# invoke function and assert result contains env vars
result = self.lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_ENV,
Payload=b'{}')
result_data = result['Payload']
self.assertEqual(result['StatusCode'], 200)
self.assertDictEqual(json.load(result_data), vars)
# get function config and assert result contains env vars
result = self.lambda_client.get_function_configuration(
FunctionName=TEST_LAMBDA_NAME_ENV)
self.assertEqual(result['Environment'], {'Variables': vars})
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_ENV)
def test_nodejs_lambda_running_in_docker(self):
if not use_docker():
return
zip_file = testutil.create_zip_file(TEST_LAMBDA_CUSTOM_RUNTIME,
get_content=True)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_CUSTOM_RUNTIME,
zip_file=zip_file,
handler='function.handler',
runtime=LAMBDA_RUNTIME_CUSTOM_RUNTIME)
result = self.lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_CUSTOM_RUNTIME,
Payload=b'{"text":"bar with \'quotes\\""}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertEqual(
to_str(result_data).strip(),
"""Echoing request: '{"text": "bar with \'quotes\\""}'""")
# assert that logs are present
expected = ['.*Custom Runtime Lambda handler executing.']
self.check_lambda_logs(TEST_LAMBDA_NAME_CUSTOM_RUNTIME,
expected_lines=expected)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_CUSTOM_RUNTIME)
def test_destroy_idle_containers(self):
# run these tests only for the "reuse containers" Lambda executor
if not isinstance(lambda_api.LAMBDA_EXECUTOR,
lambda_executors.LambdaExecutorReuseContainers):
return
executor = lambda_api.LAMBDA_EXECUTOR
func_name = 'test_destroy_idle_containers'
func_arn = lambda_api.func_arn(func_name)
# make sure existing containers are gone
executor.destroy_existing_docker_containers()
self.assertEqual(len(executor.get_all_container_names()), 0)
# deploy and invoke lambda without Docker
testutil.create_lambda_function(
func_name=func_name, handler_file=TEST_LAMBDA_ENV, libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'})
self.assertEqual(len(executor.get_all_container_names()), 0)
self.lambda_client.invoke(FunctionName=func_name, Payload=b'{}')
self.assertEqual(len(executor.get_all_container_names()), 1)
# try to destroy idle containers.
executor.idle_container_destroyer()
self.assertEqual(len(executor.get_all_container_names()), 1)
# simulate an idle container
executor.function_invoke_times[func_arn] = time.time() - lambda_executors.MAX_CONTAINER_IDLE_TIME_MS
executor.idle_container_destroyer()
self.assertEqual(len(executor.get_all_container_names()), 0)
# clean up
testutil.delete_lambda_function(func_name)
def test_lambda_start_stepfunctions_execution(self):
function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX,
short_uid())
resource_lambda_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX,
short_uid())
state_machine_name = 'state-machine-{}'.format(short_uid())
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_START_EXECUTION_FILE,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
testutil.create_lambda_function(handler_file=TEST_LAMBDA_ECHO_FILE,
func_name=resource_lambda_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
state_machine_def = {
'StartAt': 'step1',
'States': {
'step1': {
'Type': 'Task',
'Resource':
aws_stack.lambda_function_arn(resource_lambda_name),
'ResultPath': '$.result_value',
'End': True
}
}
}
sfn_client = aws_stack.connect_to_service('stepfunctions')
rs = sfn_client.create_state_machine(
name=state_machine_name,
definition=json.dumps(state_machine_def),
roleArn=aws_stack.role_arn('sfn_role'))
sm_arn = rs['stateMachineArn']
self.lambda_client.invoke(FunctionName=function_name,
Payload=json.dumps({
'state_machine_arn': sm_arn,
'region_name': config.DEFAULT_REGION,
'input': {}
}))
time.sleep(1)
rs = sfn_client.list_executions(stateMachineArn=sm_arn)
# assert that state machine get executed 1 time
self.assertEqual(
len([
ex for ex in rs['executions']
if ex['stateMachineArn'] == sm_arn
]), 1)
# clean up
testutil.delete_lambda_function(function_name)
testutil.delete_lambda_function(resource_lambda_name)
# clean up
sfn_client.delete_state_machine(stateMachineArn=sm_arn)
def test_put_events_with_target_lambda(self):
rule_name = 'rule-{}'.format(short_uid())
function_name = 'lambda-func-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
rs = testutil.create_lambda_function(handler_file=os.path.join(
THIS_FOLDER, 'lambdas', 'lambda_echo.py'),
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
func_arn = rs['CreateFunctionResponse']['FunctionArn']
self.events_client.create_event_bus(Name=TEST_EVENT_BUS_NAME)
self.events_client.put_rule(
Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
EventPattern=json.dumps(TEST_EVENT_PATTERN))
rs = self.events_client.put_targets(Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Targets=[{
'Id': target_id,
'Arn': func_arn
}])
self.assertIn('FailedEntryCount', rs)
self.assertIn('FailedEntries', rs)
self.assertEqual(rs['FailedEntryCount'], 0)
self.assertEqual(rs['FailedEntries'], [])
self.events_client.put_events(
Entries=[{
'EventBusName': TEST_EVENT_BUS_NAME,
'Source': TEST_EVENT_PATTERN['Source'],
'DetailType': TEST_EVENT_PATTERN['DetailType'],
'Detail': TEST_EVENT_PATTERN['Detail']
}])
# Get lambda's log events
events = get_lambda_log_events(function_name)
self.assertEqual(len(events), 1)
actual_event = events[0]
self.assertIsValidEvent(actual_event)
self.assertDictEqual(json.loads(actual_event['detail']),
json.loads(TEST_EVENT_PATTERN['Detail']))
# clean up
testutil.delete_lambda_function(function_name)
self.events_client.remove_targets(Rule=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Ids=[target_id],
Force=True)
self.events_client.delete_rule(Name=rule_name,
EventBusName=TEST_EVENT_BUS_NAME,
Force=True)
self.events_client.delete_event_bus(Name=TEST_EVENT_BUS_NAME)
def test_invocation_with_qualifier(self):
lambda_name = 'test_lambda_%s' % short_uid()
bucket_name = 'test_bucket_lambda2'
bucket_key = 'test_lambda.zip'
# upload zip file to S3
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27
)
self.s3_client.create_bucket(Bucket=bucket_name)
self.s3_client.upload_fileobj(
BytesIO(zip_file), bucket_name, bucket_key)
# create lambda function
response = self.lambda_client.create_function(
FunctionName=lambda_name, Handler='handler.handler',
Runtime=lambda_api.LAMBDA_RUNTIME_PYTHON27, Role='r1',
Code={
'S3Bucket': bucket_name,
'S3Key': bucket_key
},
Publish=True
)
self.assertIn('Version', response)
# invoke lambda function
data_before = b'{"foo": "bar with \'quotes\\""}'
result = self.lambda_client.invoke(
FunctionName=lambda_name,
Payload=data_before,
Qualifier=response['Version']
)
data_after = json.loads(result['Payload'].read())
self.assertEqual(json.loads(to_str(data_before)), data_after['event'])
context = data_after['context']
self.assertEqual(response['Version'], context['function_version'])
self.assertEqual(lambda_name, context['function_name'])
# assert that logs are present
expected = ['Lambda log message - print function']
if use_docker():
# Note that during regular test execution, nosetests captures the output from
# the logging module - hence we can only expect this when running in Docker
expected.append('.*Lambda log message - logging module')
self.check_lambda_logs(lambda_name, expected_lines=expected)
# clean up
testutil.delete_lambda_function(lambda_name)
def test_put_events_with_target_lambda(self):
rule_name = 'rule-{}'.format(short_uid())
function_name = 'lambda-func-{}'.format(short_uid())
target_id = 'target-{}'.format(short_uid())
bus_name = 'bus-{}'.format(short_uid())
handler_file = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_echo.py')
rs = testutil.create_lambda_function(handler_file=handler_file,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
func_arn = rs['CreateFunctionResponse']['FunctionArn']
self.events_client.create_event_bus(Name=bus_name)
self.events_client.put_rule(
Name=rule_name,
EventBusName=bus_name,
EventPattern=json.dumps(TEST_EVENT_PATTERN))
rs = self.events_client.put_targets(Rule=rule_name,
EventBusName=bus_name,
Targets=[{
'Id': target_id,
'Arn': func_arn
}])
self.assertIn('FailedEntryCount', rs)
self.assertIn('FailedEntries', rs)
self.assertEqual(0, rs['FailedEntryCount'])
self.assertEqual([], rs['FailedEntries'])
self.events_client.put_events(
Entries=[{
'EventBusName': bus_name,
'Source': TEST_EVENT_PATTERN['Source'][0],
'DetailType': TEST_EVENT_PATTERN['detail-type'][0],
'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0])
}])
# Get lambda's log events
events = retry(check_expected_lambda_log_events_length,
retries=3,
sleep=1,
function_name=function_name,
expected_length=1)
actual_event = events[0]
self.assertIsValidEvent(actual_event)
self.assertDictEqual(json.loads(actual_event['detail']),
json.loads(TEST_EVENT_PATTERN['Detail'][0]))
# clean up
testutil.delete_lambda_function(function_name)
self.cleanup(bus_name, rule_name, target_id)
def test_python3_runtime_multiple_create_with_conflicting_module(self):
lambda_client = aws_stack.create_external_boto_client("lambda")
original_do_use_docker = lambda_api.DO_USE_DOCKER
try:
# always use the local runner
lambda_api.DO_USE_DOCKER = False
python3_with_settings1 = load_file(
TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE1, mode="rb")
python3_with_settings2 = load_file(
TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE2, mode="rb")
lambda_name1 = "test1-%s" % short_uid()
testutil.create_lambda_function(
func_name=lambda_name1,
zip_file=python3_with_settings1,
runtime=LAMBDA_RUNTIME_PYTHON36,
handler="handler1.handler",
)
lambda_name2 = "test2-%s" % short_uid()
testutil.create_lambda_function(
func_name=lambda_name2,
zip_file=python3_with_settings2,
runtime=LAMBDA_RUNTIME_PYTHON36,
handler="handler2.handler",
)
result1 = lambda_client.invoke(FunctionName=lambda_name1,
Payload=b"{}")
result_data1 = result1["Payload"].read()
result2 = lambda_client.invoke(FunctionName=lambda_name2,
Payload=b"{}")
result_data2 = result2["Payload"].read()
self.assertEqual(200, result1["StatusCode"])
self.assertIn("setting1", to_str(result_data1))
self.assertEqual(200, result2["StatusCode"])
self.assertIn("setting2", to_str(result_data2))
# clean up
testutil.delete_lambda_function(lambda_name1)
testutil.delete_lambda_function(lambda_name2)
finally:
lambda_api.DO_USE_DOCKER = original_do_use_docker
def test_http_invocation_with_apigw_proxy(self):
lambda_name = "test_lambda_%s" % short_uid()
lambda_resource = "/api/v1/{proxy+}"
lambda_path = "/api/v1/hello/world"
lambda_request_context_path = "/" + TEST_STAGE_NAME + lambda_path
lambda_request_context_resource_path = lambda_resource
# create lambda function
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
func_name=lambda_name,
)
# create API Gateway and connect it to the Lambda proxy backend
lambda_uri = aws_stack.lambda_function_arn(lambda_name)
invocation_uri = "arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations"
target_uri = invocation_uri % (aws_stack.get_region(), lambda_uri)
result = testutil.connect_api_gateway_to_http_with_lambda_proxy(
"test_gateway2",
target_uri,
path=lambda_resource,
stage_name=TEST_STAGE_NAME,
)
api_id = result["id"]
url = gateway_request_url(api_id=api_id,
stage_name=TEST_STAGE_NAME,
path=lambda_path)
result = safe_requests.post(
url, data=b"{}", headers={"User-Agent": "python-requests/testing"})
content = json.loads(result.content)
self.assertEqual(lambda_path, content["path"])
self.assertEqual(lambda_resource, content["resource"])
self.assertEqual(lambda_request_context_path,
content["requestContext"]["path"])
self.assertEqual(
lambda_request_context_resource_path,
content["requestContext"]["resourcePath"],
)
# clean up
testutil.delete_lambda_function(lambda_name)
def test_ruby_lambda_running_in_docker(self):
if not use_docker():
return
zip_file = testutil.create_zip_file(TEST_LAMBDA_RUBY, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_RUBY,
zip_file=zip_file,
handler='lambda_integration.handler',
runtime=LAMBDA_RUNTIME_RUBY25)
result = self.lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_RUBY,
Payload=b'{}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertEqual(to_str(result_data).strip(), '{}')
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_RUBY)
def test_redrive_policy_lambda_subscription(self):
self.unsubscribe_all_from_sns()
lambda_name = "test-%s" % short_uid()
lambda_arn = aws_stack.lambda_function_arn(lambda_name)
testutil.create_lambda_function(
func_name=lambda_name,
libs=TEST_LAMBDA_LIBS,
handler_file=TEST_LAMBDA_PYTHON,
runtime=LAMBDA_RUNTIME_PYTHON36,
)
subscription = self.sns_client.subscribe(TopicArn=self.topic_arn,
Protocol="lambda",
Endpoint=lambda_arn)
self.sns_client.set_subscription_attributes(
SubscriptionArn=subscription["SubscriptionArn"],
AttributeName="RedrivePolicy",
AttributeValue=json.dumps({
"deadLetterTargetArn":
aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME)
}),
)
testutil.delete_lambda_function(lambda_name)
self.sns_client.publish(
TopicArn=self.topic_arn,
Message=json.dumps({"message": "test_redrive_policy"}),
)
def receive_dlq():
result = self.sqs_client.receive_message(
QueueUrl=self.dlq_url, MessageAttributeNames=["All"])
self.assertGreater(len(result["Messages"]), 0)
self.assertEqual(
json.loads(
json.loads(result["Messages"][0]["Body"])["Message"][0])
["message"],
"test_redrive_policy",
)
retry(receive_dlq, retries=10, sleep=2)
def _run_test_lambda_invoked_by_sqs_message_with_delay_seconds_dotnet(self, zip_file, handler, runtime):
if not use_docker():
return
func_name = 'dotnet-sqs-{}'.format(short_uid())
queue_name = 'queue-%s' % short_uid()
queue_url = self.client.create_queue(QueueName=queue_name)['QueueUrl']
queue_arn = aws_stack.sqs_queue_arn(queue_name)
delay_time = 1
testutil.create_lambda_function(
func_name=func_name,
zip_file=zip_file,
handler=handler,
runtime=runtime)
lambda_client = aws_stack.connect_to_service('lambda')
lambda_client.create_event_source_mapping(
EventSourceArn=queue_arn,
FunctionName=func_name
)
self.client.send_message(
QueueUrl=queue_url,
MessageBody='hello world.',
DelaySeconds=delay_time
)
# assert that the Lambda has been invoked
def get_logs():
logs = get_lambda_log_events(func_name)
self.assertGreater(len(logs), 0)
retry(get_logs, retries=5, sleep=3)
# assert that the message has been deleted from the queue
resp = self.client.receive_message(QueueUrl=queue_url, MessageAttributeNames=['All'])
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
self.assertEqual(None, resp.get('Messages', None))
# clean up
self.client.delete_queue(QueueUrl=queue_url)
testutil.delete_lambda_function(func_name)
def __run_test(self, func_name, zip_file, handler, runtime,
expected_lines):
if not use_docker():
return
testutil.create_lambda_function(func_name=func_name,
zip_file=zip_file,
handler=handler,
runtime=runtime)
result = self.lambda_client.invoke(FunctionName=func_name,
Payload=b'{}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertEqual(to_str(result_data).strip(), '{}')
# TODO make lambda log checks more resilient to various formats
# self.check_lambda_logs(func_name, expected_lines=expected_lines)
testutil.delete_lambda_function(func_name)
def test_python3_runtime_multple_create_with_conflicting_module(self):
original_do_use_docker = lambda_api.DO_USE_DOCKER
try:
# always use the local runner
lambda_api.DO_USE_DOCKER = False
python3_with_settings1 = load_file(
TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE1, mode='rb')
python3_with_settings2 = load_file(
TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE2, mode='rb')
lambda_name1 = 'test1-%s' % short_uid()
testutil.create_lambda_function(func_name=lambda_name1,
zip_file=python3_with_settings1,
runtime=LAMBDA_RUNTIME_PYTHON36,
handler='handler1.handler')
lambda_name2 = 'test2-%s' % short_uid()
testutil.create_lambda_function(func_name=lambda_name2,
zip_file=python3_with_settings2,
runtime=LAMBDA_RUNTIME_PYTHON36,
handler='handler2.handler')
result1 = self.lambda_client.invoke(FunctionName=lambda_name1,
Payload=b'{}')
result_data1 = result1['Payload'].read()
result2 = self.lambda_client.invoke(FunctionName=lambda_name2,
Payload=b'{}')
result_data2 = result2['Payload'].read()
self.assertEqual(result1['StatusCode'], 200)
self.assertIn('setting1', to_str(result_data1))
self.assertEqual(result2['StatusCode'], 200)
self.assertIn('setting2', to_str(result_data2))
# clean up
testutil.delete_lambda_function(lambda_name1)
testutil.delete_lambda_function(lambda_name2)
finally:
lambda_api.DO_USE_DOCKER = original_do_use_docker
def test_upload_lambda_from_s3(self):
lambda_name = 'test_lambda_%s' % short_uid()
bucket_name = 'test_bucket_lambda'
bucket_key = 'test_lambda.zip'
# upload zip file to S3
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
self.s3_client.create_bucket(Bucket=bucket_name)
self.s3_client.upload_fileobj(BytesIO(zip_file), bucket_name,
bucket_key)
# create lambda function
self.lambda_client.create_function(
FunctionName=lambda_name,
Handler='handler.handler',
Runtime=lambda_api.LAMBDA_RUNTIME_PYTHON27,
Role='r1',
Code={
'S3Bucket': bucket_name,
'S3Key': bucket_key
})
# invoke lambda function
data_before = b'{"foo": "bar with \'quotes\\""}'
result = self.lambda_client.invoke(FunctionName=lambda_name,
Payload=data_before)
data_after = json.loads(result['Payload'].read())
self.assertEqual(json.loads(to_str(data_before)), data_after['event'])
context = data_after['context']
self.assertEqual('$LATEST', context['function_version'])
self.assertEqual(lambda_name, context['function_name'])
# clean up
testutil.delete_lambda_function(lambda_name)
def test_destroy_idle_containers(self):
executor = lambda_api.LAMBDA_EXECUTOR
func_name = "test_destroy_idle_containers"
func_arn = lambda_api.func_arn(func_name)
# make sure existing containers are gone
executor.destroy_existing_docker_containers()
self.assertEqual(0, len(executor.get_all_container_names()))
# deploy and invoke lambda without Docker
testutil.create_lambda_function(
func_name=func_name,
handler_file=TEST_LAMBDA_ENV,
libs=TEST_LAMBDA_LIBS,
envvars={"Hello": "World"},
)
self.assertEqual(0, len(executor.get_all_container_names()))
self.lambda_client.invoke(FunctionName=func_name, Payload=b"{}")
self.assertEqual(1, len(executor.get_all_container_names()))
# try to destroy idle containers.
executor.idle_container_destroyer()
self.assertEqual(1, len(executor.get_all_container_names()))
# simulate an idle container
executor.function_invoke_times[func_arn] = (
int(time.time() * 1000) - lambda_executors.MAX_CONTAINER_IDLE_TIME_MS
)
executor.idle_container_destroyer()
def assert_container_destroyed():
self.assertEqual(0, len(executor.get_all_container_names()))
retry(assert_container_destroyed, retries=3)
# clean up
testutil.delete_lambda_function(func_name)
def test_lambda_put_item_to_dynamodb(self):
table_name = 'ddb-table-{}'.format(short_uid())
function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX,
short_uid())
aws_stack.create_dynamodb_table(table_name, partition_key='id')
testutil.create_lambda_function(handler_file=TEST_LAMBDA_PUT_ITEM_FILE,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
data = {short_uid(): 'data-{}'.format(i) for i in range(3)}
event = {
'table_name': table_name,
'region_name': config.DEFAULT_REGION,
'items': [{
'id': k,
'data': v
} for k, v in data.items()]
}
self.lambda_client.invoke(FunctionName=function_name,
Payload=json.dumps(event))
dynamodb = aws_stack.connect_to_resource('dynamodb')
rs = dynamodb.Table(table_name).scan()
items = rs['Items']
self.assertEqual(len(items), len(data.keys()))
for item in items:
self.assertEqual(data[item['id']], item['data'])
# clean up
testutil.delete_lambda_function(function_name)
dynamodb_client = aws_stack.connect_to_service('dynamodb')
dynamodb_client.delete_table(TableName=table_name)
def test_dotnet_lambda_running_in_docker(self):
if not use_docker():
return
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_DOTNETCORE2,
zip_file=self.zip_file_content,
handler='DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler',
runtime=LAMBDA_RUNTIME_DOTNETCORE2
)
result = self.lambda_client.invoke(
FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}')
result_data = result['Payload'].read()
self.assertEqual(result['StatusCode'], 200)
self.assertEqual(to_str(result_data).strip(), '{}')
# assert that logs are present
expected = ['Running .NET Core 2.0 Lambda']
self.check_lambda_logs(TEST_LAMBDA_NAME_DOTNETCORE2, expected_lines=expected)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DOTNETCORE2)
def test_lambda_send_message_to_sqs(self):
function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX,
short_uid())
queue_name = 'lambda-queue-{}'.format(short_uid())
sqs_client = aws_stack.connect_to_service('sqs')
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_SEND_MESSAGE_FILE,
func_name=function_name,
runtime=LAMBDA_RUNTIME_PYTHON36)
queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl']
event = {
'message': 'message-from-test-lambda-{}'.format(short_uid()),
'queue_name': queue_name,
'region_name': config.DEFAULT_REGION
}
self.lambda_client.invoke(FunctionName=function_name,
Payload=json.dumps(event))
# assert that message has been received on the Queue
def receive_message():
rs = sqs_client.receive_message(QueueUrl=queue_url,
MessageAttributeNames=['All'])
self.assertGreater(len(rs['Messages']), 0)
return rs['Messages'][0]
message = retry(receive_message, retries=3, sleep=2)
self.assertEqual(message['Body'], event['message'])
# clean up
testutil.delete_lambda_function(function_name)
sqs_client.delete_queue(QueueUrl=queue_url)
def setup_and_tear_down():
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_ENV),
get_content=True)
zip_file2 = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON_ECHO), get_content=True)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_1,
zip_file=zip_file,
envvars={"Hello": TEST_RESULT_VALUE},
)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_2,
zip_file=zip_file,
envvars={"Hello": TEST_RESULT_VALUE},
)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_3,
zip_file=zip_file,
envvars={"Hello": "Replace Value"},
)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_4,
zip_file=zip_file,
envvars={"Hello": TEST_RESULT_VALUE},
)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_5,
zip_file=zip_file2)
yield
testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_1)
testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_2)
testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_3)
testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_4)
testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_5)
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = '-kclapp2'
table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_service('dynamodb', client=True)
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27,
delete=True)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId0'
},
'data': {
'S': 'foobar123'
}
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId1'
},
'data': {
'S': 'foobar123'
}
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId2'
},
'data': {
'S': 'foobar123'
}
}
}
}]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId3'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId4'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'PutRequest': {
'Item': {
PARTITION_KEY: {
'S': 'testId5'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId0'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId1'
}
}
}
},
{
'DeleteRequest': {
'Key': {
PARTITION_KEY: {
'S': 'testId2'
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId6'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId7'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId8'
},
'data': {
'S': 'foobar123'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId3'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId4'
}
}
}
},
{
'Delete': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId5'
}
}
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId6'
},
'data': {
'S': 'foobar123_updated1'
}
}
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
'Put': {
'TableName': table_name,
'Item': {
PARTITION_KEY: {
'S': 'testId7'
},
'data': {
'S': 'foobar123_updated1'
}
}
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId6'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId7'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
{
'Update': {
'TableName': table_name,
'Key': {
PARTITION_KEY: {
'S': 'testId8'
}
},
'UpdateExpression': 'SET #0 = :0',
'ExpressionAttributeNames': {
'#0': 'data'
},
'ExpressionAttributeValues': {
':0': {
'S': 'foobar123_updated2'
}
}
}
},
])
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
LOGGER.warning(
('DynamoDB updates retrieved (actual/expected): %s/%s') %
(len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [
json.loads(base64.b64decode(e['data'])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
removes = [
e for e in event_items if e.get('__action_type') == 'REMOVE'
]
self.assertEqual(len(inserts), num_insert)
self.assertEqual(len(modifies), num_modify)
self.assertEqual(len(removes), num_delete)
for i, event in enumerate(inserts):
self.assertNotIn('old_image', event)
self.assertEqual(inserts[i]['new_image'], {
'id': 'testId%d' % i,
'data': 'foobar123'
})
self.assertEqual(modifies[0]['old_image'], {
'id': 'testId6',
'data': 'foobar123'
})
self.assertEqual(modifies[0]['new_image'], {
'id': 'testId6',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[1]['old_image'], {
'id': 'testId7',
'data': 'foobar123'
})
self.assertEqual(modifies[1]['new_image'], {
'id': 'testId7',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[2]['old_image'], {
'id': 'testId6',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[2]['new_image'], {
'id': 'testId6',
'data': 'foobar123_updated2'
})
self.assertEqual(modifies[3]['old_image'], {
'id': 'testId7',
'data': 'foobar123_updated1'
})
self.assertEqual(modifies[3]['new_image'], {
'id': 'testId7',
'data': 'foobar123_updated2'
})
self.assertEqual(modifies[4]['old_image'], {
'id': 'testId8',
'data': 'foobar123'
})
self.assertEqual(modifies[4]['new_image'], {
'id': 'testId8',
'data': 'foobar123_updated2'
})
for i, event in enumerate(removes):
self.assertEqual(event['old_image'], {
'id': 'testId%d' % i,
'data': 'foobar123'
})
self.assertNotIn('new_image', event)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_kinesis_lambda_sns_ddb_sqs_streams(self):
ddb_lease_table_suffix = '-kclapp'
table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
dynamodb = aws_stack.connect_to_resource('dynamodb')
dynamodb_service = aws_stack.connect_to_service('dynamodb')
dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams')
kinesis = aws_stack.connect_to_service('kinesis')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
LOGGER.info('Creating test streams...')
run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False)
aws_stack.create_kinesis_stream(stream_name, delete=True)
aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix)
LOGGER.info('Kinesis consumer initialized.')
# create table with stream forwarding config
aws_stack.create_dynamodb_table(table_name,
partition_key=PARTITION_KEY,
stream_view_type='NEW_AND_OLD_IMAGES')
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams['Streams']:
if stream['TableName'] == table_name:
ddb_event_source_arn = stream['StreamArn']
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
zip_file = testutil.create_lambda_archive(
load_file(TEST_LAMBDA_PYTHON),
get_content=True,
libs=TEST_LAMBDA_LIBS,
runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27,
delete=True)
# make sure we cannot create Lambda with same name twice
assert_raises(Exception,
testutil.create_lambda_function,
func_name=TEST_LAMBDA_NAME_DDB,
zip_file=zip_file,
event_source_arn=ddb_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to Kinesis Stream
kinesis_event_source_arn = kinesis.describe_stream(
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME
)['StreamDescription']['StreamARN']
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_STREAM,
zip_file=zip_file,
event_source_arn=kinesis_event_source_arn,
runtime=LAMBDA_RUNTIME_PYTHON27)
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_QUEUE,
zip_file=zip_file,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27)
# set number of items to update/put to table
num_events_ddb = 15
num_put_new_items = 5
num_put_existing_items = 2
num_batch_items = 3
num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items
LOGGER.info('Putting %s items to table...' % num_events_ddb)
table = dynamodb.Table(table_name)
for i in range(0, num_put_new_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123'
})
# Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522)
for i in range(0, num_put_existing_items):
table.put_item(Item={
PARTITION_KEY: 'testId%s' % i,
'data': 'foobar123_put_existing'
})
# batch write some items containing non-ASCII characters
dynamodb.batch_write_item(
RequestItems={
table_name: [{
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ✓'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 £'
}
}
}, {
'PutRequest': {
'Item': {
PARTITION_KEY: short_uid(),
'data': 'foobar123 ¢'
}
}
}]
})
# update some items, which also triggers notification events
for i in range(0, num_updates_ddb):
dynamodb_service.update_item(
TableName=table_name,
Key={PARTITION_KEY: {
'S': 'testId%s' % i
}},
AttributeUpdates={
'data': {
'Action': 'PUT',
'Value': {
'S': 'foobar123_updated'
}
}
})
# put items to stream
num_events_kinesis = 10
LOGGER.info('Putting %s items to stream...' % num_events_kinesis)
kinesis.put_records(Records=[{
'Data': '{}',
'PartitionKey': 'testId%s' % i
} for i in range(0, num_events_kinesis)],
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# put 1 item to stream that will trigger an error in the Lambda
kinesis.put_record(Data='{"%s": 1}' %
lambda_integration.MSG_BODY_RAISE_ERROR_FLAG,
PartitionKey='testIderror',
StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)
# create SNS topic, connect it to the Lambda, publish test messages
num_events_sns = 3
response = sns.create_topic(Name=TEST_TOPIC_NAME)
sns.subscribe(
TopicArn=response['TopicArn'],
Protocol='lambda',
Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM))
for i in range(0, num_events_sns):
sns.publish(TopicArn=response['TopicArn'],
Message='test message %s' % i)
# get latest records
latest = aws_stack.kinesis_get_latest_records(
TEST_LAMBDA_SOURCE_STREAM_NAME,
shard_id='shardId-000000000000',
count=10)
self.assertEqual(len(latest), 10)
# send messages to SQS queue
num_events_sqs = 4
for i in range(num_events_sqs):
sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'],
MessageBody=str(i))
LOGGER.info('Waiting some time before finishing test.')
time.sleep(2)
num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs
num_events = num_events_lambda + num_events_kinesis
def check_events():
if len(events) != num_events:
LOGGER.warning((
'DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s'
) % (len(events), num_events))
self.assertEqual(len(events), num_events)
event_items = [
json.loads(base64.b64decode(e['data'])) for e in events
]
# make sure the we have the right amount of INSERT/MODIFY event types
inserts = [
e for e in event_items if e.get('__action_type') == 'INSERT'
]
modifies = [
e for e in event_items if e.get('__action_type') == 'MODIFY'
]
self.assertEqual(len(inserts), num_put_new_items + num_batch_items)
self.assertEqual(len(modifies),
num_put_existing_items + num_updates_ddb)
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=3)
# check cloudwatch notifications
num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM)
# TODO: It seems that CloudWatch is currently reporting an incorrect number of
# invocations, namely the sum over *all* lambdas, not the single one we're asking for.
# Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single
# Lambda invocation may happen with a set of Kinesis records, hence we cannot simply
# add num_events_ddb to num_events_lambda above!
# self.assertEqual(num_invocations, 2 + num_events_lambda)
self.assertGreater(num_invocations, num_events_sns + num_events_sqs)
num_error_invocations = get_lambda_invocations_count(
TEST_LAMBDA_NAME_STREAM, 'Errors')
self.assertEqual(num_error_invocations, 1)
# clean up
testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM)
testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_sqs_batch_lambda_forward(self):
sqs = aws_stack.connect_to_service('sqs')
lambda_api = aws_stack.connect_to_service('lambda')
lambda_name_queue_batch = 'lambda_queue_batch-%s' % short_uid()
# deploy test lambda connected to SQS queue
sqs_queue_info = testutil.create_sqs_queue(lambda_name_queue_batch)
queue_url = sqs_queue_info['QueueUrl']
resp = testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON_ECHO,
func_name=lambda_name_queue_batch,
event_source_arn=sqs_queue_info['QueueArn'],
runtime=LAMBDA_RUNTIME_PYTHON27,
libs=TEST_LAMBDA_LIBS)
event_source_id = resp['CreateEventSourceMappingResponse']['UUID']
lambda_api.update_event_source_mapping(UUID=event_source_id,
BatchSize=5)
messages_to_send = [{
'Id': 'message{:02d}'.format(i),
'MessageBody': 'msgBody{:02d}'.format(i),
'MessageAttributes': {
'CustomAttribute': {
'DataType': 'String',
'StringValue': 'CustomAttributeValue{:02d}'.format(i)
}
}
} for i in range(1, 12)]
# send 11 messages (which should get split into 3 batches)
sqs.send_message_batch(QueueUrl=queue_url,
Entries=messages_to_send[:10])
sqs.send_message(
QueueUrl=queue_url,
MessageBody=messages_to_send[10]['MessageBody'],
MessageAttributes=messages_to_send[10]['MessageAttributes'])
def wait_for_done():
attributes = sqs.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=[
'ApproximateNumberOfMessages',
'ApproximateNumberOfMessagesDelayed',
'ApproximateNumberOfMessagesNotVisible'
],
)['Attributes']
msg_count = int(attributes.get('ApproximateNumberOfMessages'))
self.assertEqual(msg_count, 0, 'expecting queue to be empty')
delayed_count = int(
attributes.get('ApproximateNumberOfMessagesDelayed'))
if delayed_count != 0:
LOGGER.warning(
'SQS delayed message count (actual/expected): %s/%s' %
(delayed_count, 0))
not_visible_count = int(
attributes.get('ApproximateNumberOfMessagesNotVisible'))
if not_visible_count != 0:
LOGGER.warning(
'SQS messages not visible (actual/expected): %s/%s' %
(not_visible_count, 0))
self.assertEqual(delayed_count, 0, 'no messages waiting for retry')
self.assertEqual(delayed_count + not_visible_count, 0,
'no in flight messages')
# wait for the queue to drain (max 60s)
retry(wait_for_done, retries=12, sleep=5.0)
events = get_lambda_log_events(lambda_name_queue_batch, 10)
self.assertEqual(len(events), 3, 'expected 3 lambda invocations')
testutil.delete_lambda_function(lambda_name_queue_batch)
sqs.delete_queue(QueueUrl=queue_url)
def tearDownClass(cls):
testutil.delete_lambda_function(cls.scheduled_lambda_name)
def test_lambda_streams_batch_and_transactions(self):
ddb_lease_table_suffix = "-kclapp2"
table_name = TEST_TABLE_NAME + "lsbat" + ddb_lease_table_suffix
stream_name = TEST_STREAM_NAME
lambda_ddb_name = "lambda-ddb-%s" % short_uid()
dynamodb = aws_stack.create_external_boto_client("dynamodb",
client=True)
dynamodb_service = aws_stack.create_external_boto_client("dynamodb")
dynamodbstreams = aws_stack.create_external_boto_client(
"dynamodbstreams")
LOGGER.info("Creating test streams...")
run_safe(
lambda: dynamodb_service.delete_table(TableName=stream_name +
ddb_lease_table_suffix),
print_error=False,
)
aws_stack.create_kinesis_stream(stream_name, delete=True)
events = []
# subscribe to inbound Kinesis stream
def process_records(records, shard_id):
events.extend(records)
# start the KCL client process in the background
kinesis_connector.listen_to_kinesis(
stream_name,
listener_func=process_records,
wait_until_started=True,
ddb_lease_table_suffix=ddb_lease_table_suffix,
)
LOGGER.info("Kinesis consumer initialized.")
# create table with stream forwarding config
aws_stack.create_dynamodb_table(
table_name,
partition_key=PARTITION_KEY,
stream_view_type="NEW_AND_OLD_IMAGES",
)
# list DDB streams and make sure the table stream is there
streams = dynamodbstreams.list_streams()
ddb_event_source_arn = None
for stream in streams["Streams"]:
if stream["TableName"] == table_name:
ddb_event_source_arn = stream["StreamArn"]
self.assertTrue(ddb_event_source_arn)
# deploy test lambda connected to DynamoDB Stream
testutil.create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
libs=TEST_LAMBDA_LIBS,
func_name=lambda_ddb_name,
event_source_arn=ddb_event_source_arn,
delete=True,
)
# submit a batch with writes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId0"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId1"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId2"
},
"data": {
"S": "foobar123"
},
}
}
},
]
})
# submit a batch with writes and deletes
dynamodb.batch_write_item(
RequestItems={
table_name: [
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId3"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId4"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"PutRequest": {
"Item": {
PARTITION_KEY: {
"S": "testId5"
},
"data": {
"S": "foobar123"
},
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId0"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId1"
}
}
}
},
{
"DeleteRequest": {
"Key": {
PARTITION_KEY: {
"S": "testId2"
}
}
}
},
]
})
# submit a transaction with writes and delete
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId8"
},
"data": {
"S": "foobar123"
},
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId3"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId4"
}
},
}
},
{
"Delete": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId5"
}
},
}
},
])
# submit a batch with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId6"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with a put over existing item
dynamodb.transact_write_items(TransactItems=[
{
"Put": {
"TableName": table_name,
"Item": {
PARTITION_KEY: {
"S": "testId7"
},
"data": {
"S": "foobar123_updated1"
},
},
}
},
])
# submit a transaction with updates
dynamodb.transact_write_items(TransactItems=[
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId6"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId7"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
{
"Update": {
"TableName": table_name,
"Key": {
PARTITION_KEY: {
"S": "testId8"
}
},
"UpdateExpression": "SET #0 = :0",
"ExpressionAttributeNames": {
"#0": "data"
},
"ExpressionAttributeValues": {
":0": {
"S": "foobar123_updated2"
}
},
}
},
])
LOGGER.info("Waiting some time before finishing test.")
time.sleep(2)
num_insert = 9
num_modify = 5
num_delete = 6
num_events = num_insert + num_modify + num_delete
def check_events():
if len(events) != num_events:
msg = "DynamoDB updates retrieved (actual/expected): %s/%s" % (
len(events),
num_events,
)
LOGGER.warning(msg)
self.assertEqual(num_events, len(events))
event_items = [
json.loads(base64.b64decode(e["data"])) for e in events
]
# make sure the we have the right amount of expected event types
inserts = [
e for e in event_items if e.get("__action_type") == "INSERT"
]
modifies = [
e for e in event_items if e.get("__action_type") == "MODIFY"
]
removes = [
e for e in event_items if e.get("__action_type") == "REMOVE"
]
self.assertEqual(num_insert, len(inserts))
self.assertEqual(num_modify, len(modifies))
self.assertEqual(num_delete, len(removes))
# assert that all inserts were received
for i, event in enumerate(inserts):
self.assertNotIn("old_image", event)
item_id = "testId%d" % i
matching = [
i for i in inserts if i["new_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["new_image"])
# assert that all updates were received
def assert_updates(expected_updates, modifies):
def found(update):
for modif in modifies:
if modif["old_image"]["id"] == update["id"]:
self.assertEqual(
modif["old_image"],
{
"id": update["id"],
"data": update["old"]
},
)
self.assertEqual(
modif["new_image"],
{
"id": update["id"],
"data": update["new"]
},
)
return True
for update in expected_updates:
self.assertTrue(found(update))
updates1 = [
{
"id": "testId6",
"old": "foobar123",
"new": "foobar123_updated1"
},
{
"id": "testId7",
"old": "foobar123",
"new": "foobar123_updated1"
},
]
updates2 = [
{
"id": "testId6",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId7",
"old": "foobar123_updated1",
"new": "foobar123_updated2",
},
{
"id": "testId8",
"old": "foobar123",
"new": "foobar123_updated2"
},
]
assert_updates(updates1, modifies[:2])
assert_updates(updates2, modifies[2:])
# assert that all removes were received
for i, event in enumerate(removes):
self.assertNotIn("new_image", event)
item_id = "testId%d" % i
matching = [
i for i in removes if i["old_image"]["id"] == item_id
][0]
self.assertEqual({
"id": item_id,
"data": "foobar123"
}, matching["old_image"])
# this can take a long time in CI, make sure we give it enough time/retries
retry(check_events, retries=9, sleep=4)
# clean up
testutil.delete_lambda_function(lambda_ddb_name)
