def test_python_lambda_running_in_docker(self): if not use_docker(): return zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON3), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36 ) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_PY3, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36 ) result = self.lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertEqual(to_str(result_data).strip(), '{}') # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_PY3)
def test_java_runtime_with_lib(self): java_jar_with_lib = load_file(TEST_LAMBDA_JAVA_WITH_LIB, mode='rb') # create ZIP file from JAR file jar_dir = new_tmp_dir() zip_dir = new_tmp_dir() unzip(TEST_LAMBDA_JAVA_WITH_LIB, jar_dir) shutil.move(os.path.join(jar_dir, 'lib'), os.path.join(zip_dir, 'lib')) jar_without_libs_file = testutil.create_zip_file(jar_dir) shutil.copy(jar_without_libs_file, os.path.join(zip_dir, 'lib', 'lambda.jar')) java_zip_with_lib = testutil.create_zip_file(zip_dir, get_content=True) for archive in [java_jar_with_lib, java_zip_with_lib]: lambda_name = 'test-%s' % short_uid() testutil.create_lambda_function( func_name=lambda_name, zip_file=archive, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandlerWithLib') result = self.lambda_client.invoke(FunctionName=lambda_name, Payload=b'{"echo":"echo"}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertIn('echo', to_str(result_data)) # clean up testutil.delete_lambda_function(lambda_name)
def test_nodejs_lambda_running_in_docker(self): if not use_docker(): return zip_file = testutil.create_zip_file( TEST_LAMBDA_NODEJS, get_content=True) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JS, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS810 ) result = self.lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertEqual(to_str(result_data).strip(), '{}') # assert that logs are present expected = ['.*Node.js Lambda handler executing.'] self.check_lambda_logs(TEST_LAMBDA_NAME_JS, expected_lines=expected) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_JS)
def test_put_events_with_target_lambda(self): rule_name = "rule-{}".format(short_uid()) function_name = "lambda-func-{}".format(short_uid()) target_id = "target-{}".format(short_uid()) bus_name = "bus-{}".format(short_uid()) rs = testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON_ECHO, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36, ) func_arn = rs["CreateFunctionResponse"]["FunctionArn"] self.events_client.create_event_bus(Name=bus_name) self.events_client.put_rule( Name=rule_name, EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN), ) rs = self.events_client.put_targets( Rule=rule_name, EventBusName=bus_name, Targets=[{ "Id": target_id, "Arn": func_arn }], ) self.assertIn("FailedEntryCount", rs) self.assertIn("FailedEntries", rs) self.assertEqual(0, rs["FailedEntryCount"]) self.assertEqual([], rs["FailedEntries"]) self.events_client.put_events( Entries=[{ "EventBusName": bus_name, "Source": TEST_EVENT_PATTERN["Source"][0], "DetailType": TEST_EVENT_PATTERN["detail-type"][0], "Detail": json.dumps(TEST_EVENT_PATTERN["Detail"][0]), }]) # Get lambda's log events events = retry( check_expected_lambda_log_events_length, retries=3, sleep=1, function_name=function_name, expected_length=1, ) actual_event = events[0] self.assertIsValidEvent(actual_event) self.assertDictEqual( json.loads(actual_event["detail"]), json.loads(TEST_EVENT_PATTERN["Detail"][0]), ) # clean up testutil.delete_lambda_function(function_name) self.cleanup(bus_name, rule_name, target_id)
def test_redrive_policy_lambda_subscription(self): self.unsubscribe_all_from_sns() lambda_name = 'test-%s' % short_uid() lambda_arn = aws_stack.lambda_function_arn(lambda_name) testutil.create_lambda_function(func_name=lambda_name, libs=TEST_LAMBDA_LIBS, handler_file=TEST_LAMBDA_PYTHON, runtime=LAMBDA_RUNTIME_PYTHON36) subscription = self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol='lambda', Endpoint=lambda_arn) self.sns_client.set_subscription_attributes( SubscriptionArn=subscription['SubscriptionArn'], AttributeName='RedrivePolicy', AttributeValue=json.dumps({'deadLetterTargetArn': aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME)}) ) testutil.delete_lambda_function(lambda_name) self.sns_client.publish(TopicArn=self.topic_arn, Message=json.dumps({'message': 'test_redrive_policy'})) def receive_dlq(): result = self.sqs_client.receive_message(QueueUrl=self.dlq_url, MessageAttributeNames=['All']) self.assertGreater(len(result['Messages']), 0) self.assertEqual( json.loads(json.loads(result['Messages'][0]['Body'])['Message'][0])['message'], 'test_redrive_policy' ) retry(receive_dlq, retries=10, sleep=2)
def test_lambda_environment(self): vars = {'Hello': 'World'} zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_ENV), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_ENV, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27, envvars=vars) # invoke function and assert result contains env vars result = self.lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_ENV, Payload=b'{}') result_data = result['Payload'] self.assertEqual(result['StatusCode'], 200) self.assertDictEqual(json.load(result_data), vars) # get function config and assert result contains env vars result = self.lambda_client.get_function_configuration( FunctionName=TEST_LAMBDA_NAME_ENV) self.assertEqual(result['Environment'], {'Variables': vars}) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_ENV)
def test_nodejs_lambda_running_in_docker(self): if not use_docker(): return zip_file = testutil.create_zip_file(TEST_LAMBDA_CUSTOM_RUNTIME, get_content=True) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_CUSTOM_RUNTIME, zip_file=zip_file, handler='function.handler', runtime=LAMBDA_RUNTIME_CUSTOM_RUNTIME) result = self.lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_CUSTOM_RUNTIME, Payload=b'{"text":"bar with \'quotes\\""}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertEqual( to_str(result_data).strip(), """Echoing request: '{"text": "bar with \'quotes\\""}'""") # assert that logs are present expected = ['.*Custom Runtime Lambda handler executing.'] self.check_lambda_logs(TEST_LAMBDA_NAME_CUSTOM_RUNTIME, expected_lines=expected) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_CUSTOM_RUNTIME)
def test_destroy_idle_containers(self): # run these tests only for the "reuse containers" Lambda executor if not isinstance(lambda_api.LAMBDA_EXECUTOR, lambda_executors.LambdaExecutorReuseContainers): return executor = lambda_api.LAMBDA_EXECUTOR func_name = 'test_destroy_idle_containers' func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone executor.destroy_existing_docker_containers() self.assertEqual(len(executor.get_all_container_names()), 0) # deploy and invoke lambda without Docker testutil.create_lambda_function( func_name=func_name, handler_file=TEST_LAMBDA_ENV, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'}) self.assertEqual(len(executor.get_all_container_names()), 0) self.lambda_client.invoke(FunctionName=func_name, Payload=b'{}') self.assertEqual(len(executor.get_all_container_names()), 1) # try to destroy idle containers. executor.idle_container_destroyer() self.assertEqual(len(executor.get_all_container_names()), 1) # simulate an idle container executor.function_invoke_times[func_arn] = time.time() - lambda_executors.MAX_CONTAINER_IDLE_TIME_MS executor.idle_container_destroyer() self.assertEqual(len(executor.get_all_container_names()), 0) # clean up testutil.delete_lambda_function(func_name)
def test_lambda_start_stepfunctions_execution(self): function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX, short_uid()) resource_lambda_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX, short_uid()) state_machine_name = 'state-machine-{}'.format(short_uid()) testutil.create_lambda_function( handler_file=TEST_LAMBDA_START_EXECUTION_FILE, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36) testutil.create_lambda_function(handler_file=TEST_LAMBDA_ECHO_FILE, func_name=resource_lambda_name, runtime=LAMBDA_RUNTIME_PYTHON36) state_machine_def = { 'StartAt': 'step1', 'States': { 'step1': { 'Type': 'Task', 'Resource': aws_stack.lambda_function_arn(resource_lambda_name), 'ResultPath': '$.result_value', 'End': True } } } sfn_client = aws_stack.connect_to_service('stepfunctions') rs = sfn_client.create_state_machine( name=state_machine_name, definition=json.dumps(state_machine_def), roleArn=aws_stack.role_arn('sfn_role')) sm_arn = rs['stateMachineArn'] self.lambda_client.invoke(FunctionName=function_name, Payload=json.dumps({ 'state_machine_arn': sm_arn, 'region_name': config.DEFAULT_REGION, 'input': {} })) time.sleep(1) rs = sfn_client.list_executions(stateMachineArn=sm_arn) # assert that state machine get executed 1 time self.assertEqual( len([ ex for ex in rs['executions'] if ex['stateMachineArn'] == sm_arn ]), 1) # clean up testutil.delete_lambda_function(function_name) testutil.delete_lambda_function(resource_lambda_name) # clean up sfn_client.delete_state_machine(stateMachineArn=sm_arn)
def test_put_events_with_target_lambda(self): rule_name = 'rule-{}'.format(short_uid()) function_name = 'lambda-func-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) rs = testutil.create_lambda_function(handler_file=os.path.join( THIS_FOLDER, 'lambdas', 'lambda_echo.py'), func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36) func_arn = rs['CreateFunctionResponse']['FunctionArn'] self.events_client.create_event_bus(Name=TEST_EVENT_BUS_NAME) self.events_client.put_rule( Name=rule_name, EventBusName=TEST_EVENT_BUS_NAME, EventPattern=json.dumps(TEST_EVENT_PATTERN)) rs = self.events_client.put_targets(Rule=rule_name, EventBusName=TEST_EVENT_BUS_NAME, Targets=[{ 'Id': target_id, 'Arn': func_arn }]) self.assertIn('FailedEntryCount', rs) self.assertIn('FailedEntries', rs) self.assertEqual(rs['FailedEntryCount'], 0) self.assertEqual(rs['FailedEntries'], []) self.events_client.put_events( Entries=[{ 'EventBusName': TEST_EVENT_BUS_NAME, 'Source': TEST_EVENT_PATTERN['Source'], 'DetailType': TEST_EVENT_PATTERN['DetailType'], 'Detail': TEST_EVENT_PATTERN['Detail'] }]) # Get lambda's log events events = get_lambda_log_events(function_name) self.assertEqual(len(events), 1) actual_event = events[0] self.assertIsValidEvent(actual_event) self.assertDictEqual(json.loads(actual_event['detail']), json.loads(TEST_EVENT_PATTERN['Detail'])) # clean up testutil.delete_lambda_function(function_name) self.events_client.remove_targets(Rule=rule_name, EventBusName=TEST_EVENT_BUS_NAME, Ids=[target_id], Force=True) self.events_client.delete_rule(Name=rule_name, EventBusName=TEST_EVENT_BUS_NAME, Force=True) self.events_client.delete_event_bus(Name=TEST_EVENT_BUS_NAME)
def test_invocation_with_qualifier(self): lambda_name = 'test_lambda_%s' % short_uid() bucket_name = 'test_bucket_lambda2' bucket_key = 'test_lambda.zip' # upload zip file to S3 zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27 ) self.s3_client.create_bucket(Bucket=bucket_name) self.s3_client.upload_fileobj( BytesIO(zip_file), bucket_name, bucket_key) # create lambda function response = self.lambda_client.create_function( FunctionName=lambda_name, Handler='handler.handler', Runtime=lambda_api.LAMBDA_RUNTIME_PYTHON27, Role='r1', Code={ 'S3Bucket': bucket_name, 'S3Key': bucket_key }, Publish=True ) self.assertIn('Version', response) # invoke lambda function data_before = b'{"foo": "bar with \'quotes\\""}' result = self.lambda_client.invoke( FunctionName=lambda_name, Payload=data_before, Qualifier=response['Version'] ) data_after = json.loads(result['Payload'].read()) self.assertEqual(json.loads(to_str(data_before)), data_after['event']) context = data_after['context'] self.assertEqual(response['Version'], context['function_version']) self.assertEqual(lambda_name, context['function_name']) # assert that logs are present expected = ['Lambda log message - print function'] if use_docker(): # Note that during regular test execution, nosetests captures the output from # the logging module - hence we can only expect this when running in Docker expected.append('.*Lambda log message - logging module') self.check_lambda_logs(lambda_name, expected_lines=expected) # clean up testutil.delete_lambda_function(lambda_name)
def test_put_events_with_target_lambda(self): rule_name = 'rule-{}'.format(short_uid()) function_name = 'lambda-func-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) bus_name = 'bus-{}'.format(short_uid()) handler_file = os.path.join(THIS_FOLDER, 'lambdas', 'lambda_echo.py') rs = testutil.create_lambda_function(handler_file=handler_file, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36) func_arn = rs['CreateFunctionResponse']['FunctionArn'] self.events_client.create_event_bus(Name=bus_name) self.events_client.put_rule( Name=rule_name, EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN)) rs = self.events_client.put_targets(Rule=rule_name, EventBusName=bus_name, Targets=[{ 'Id': target_id, 'Arn': func_arn }]) self.assertIn('FailedEntryCount', rs) self.assertIn('FailedEntries', rs) self.assertEqual(0, rs['FailedEntryCount']) self.assertEqual([], rs['FailedEntries']) self.events_client.put_events( Entries=[{ 'EventBusName': bus_name, 'Source': TEST_EVENT_PATTERN['Source'][0], 'DetailType': TEST_EVENT_PATTERN['detail-type'][0], 'Detail': json.dumps(TEST_EVENT_PATTERN['Detail'][0]) }]) # Get lambda's log events events = retry(check_expected_lambda_log_events_length, retries=3, sleep=1, function_name=function_name, expected_length=1) actual_event = events[0] self.assertIsValidEvent(actual_event) self.assertDictEqual(json.loads(actual_event['detail']), json.loads(TEST_EVENT_PATTERN['Detail'][0])) # clean up testutil.delete_lambda_function(function_name) self.cleanup(bus_name, rule_name, target_id)
def test_python3_runtime_multiple_create_with_conflicting_module(self): lambda_client = aws_stack.create_external_boto_client("lambda") original_do_use_docker = lambda_api.DO_USE_DOCKER try: # always use the local runner lambda_api.DO_USE_DOCKER = False python3_with_settings1 = load_file( TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE1, mode="rb") python3_with_settings2 = load_file( TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE2, mode="rb") lambda_name1 = "test1-%s" % short_uid() testutil.create_lambda_function( func_name=lambda_name1, zip_file=python3_with_settings1, runtime=LAMBDA_RUNTIME_PYTHON36, handler="handler1.handler", ) lambda_name2 = "test2-%s" % short_uid() testutil.create_lambda_function( func_name=lambda_name2, zip_file=python3_with_settings2, runtime=LAMBDA_RUNTIME_PYTHON36, handler="handler2.handler", ) result1 = lambda_client.invoke(FunctionName=lambda_name1, Payload=b"{}") result_data1 = result1["Payload"].read() result2 = lambda_client.invoke(FunctionName=lambda_name2, Payload=b"{}") result_data2 = result2["Payload"].read() self.assertEqual(200, result1["StatusCode"]) self.assertIn("setting1", to_str(result_data1)) self.assertEqual(200, result2["StatusCode"]) self.assertIn("setting2", to_str(result_data2)) # clean up testutil.delete_lambda_function(lambda_name1) testutil.delete_lambda_function(lambda_name2) finally: lambda_api.DO_USE_DOCKER = original_do_use_docker
def test_http_invocation_with_apigw_proxy(self): lambda_name = "test_lambda_%s" % short_uid() lambda_resource = "/api/v1/{proxy+}" lambda_path = "/api/v1/hello/world" lambda_request_context_path = "/" + TEST_STAGE_NAME + lambda_path lambda_request_context_resource_path = lambda_resource # create lambda function testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, libs=TEST_LAMBDA_LIBS, func_name=lambda_name, ) # create API Gateway and connect it to the Lambda proxy backend lambda_uri = aws_stack.lambda_function_arn(lambda_name) invocation_uri = "arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations" target_uri = invocation_uri % (aws_stack.get_region(), lambda_uri) result = testutil.connect_api_gateway_to_http_with_lambda_proxy( "test_gateway2", target_uri, path=lambda_resource, stage_name=TEST_STAGE_NAME, ) api_id = result["id"] url = gateway_request_url(api_id=api_id, stage_name=TEST_STAGE_NAME, path=lambda_path) result = safe_requests.post( url, data=b"{}", headers={"User-Agent": "python-requests/testing"}) content = json.loads(result.content) self.assertEqual(lambda_path, content["path"]) self.assertEqual(lambda_resource, content["resource"]) self.assertEqual(lambda_request_context_path, content["requestContext"]["path"]) self.assertEqual( lambda_request_context_resource_path, content["requestContext"]["resourcePath"], ) # clean up testutil.delete_lambda_function(lambda_name)
def test_ruby_lambda_running_in_docker(self): if not use_docker(): return zip_file = testutil.create_zip_file(TEST_LAMBDA_RUBY, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_RUBY, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_RUBY25) result = self.lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_RUBY, Payload=b'{}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertEqual(to_str(result_data).strip(), '{}') # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_RUBY)
def test_redrive_policy_lambda_subscription(self): self.unsubscribe_all_from_sns() lambda_name = "test-%s" % short_uid() lambda_arn = aws_stack.lambda_function_arn(lambda_name) testutil.create_lambda_function( func_name=lambda_name, libs=TEST_LAMBDA_LIBS, handler_file=TEST_LAMBDA_PYTHON, runtime=LAMBDA_RUNTIME_PYTHON36, ) subscription = self.sns_client.subscribe(TopicArn=self.topic_arn, Protocol="lambda", Endpoint=lambda_arn) self.sns_client.set_subscription_attributes( SubscriptionArn=subscription["SubscriptionArn"], AttributeName="RedrivePolicy", AttributeValue=json.dumps({ "deadLetterTargetArn": aws_stack.sqs_queue_arn(TEST_QUEUE_DLQ_NAME) }), ) testutil.delete_lambda_function(lambda_name) self.sns_client.publish( TopicArn=self.topic_arn, Message=json.dumps({"message": "test_redrive_policy"}), ) def receive_dlq(): result = self.sqs_client.receive_message( QueueUrl=self.dlq_url, MessageAttributeNames=["All"]) self.assertGreater(len(result["Messages"]), 0) self.assertEqual( json.loads( json.loads(result["Messages"][0]["Body"])["Message"][0]) ["message"], "test_redrive_policy", ) retry(receive_dlq, retries=10, sleep=2)
def _run_test_lambda_invoked_by_sqs_message_with_delay_seconds_dotnet(self, zip_file, handler, runtime): if not use_docker(): return func_name = 'dotnet-sqs-{}'.format(short_uid()) queue_name = 'queue-%s' % short_uid() queue_url = self.client.create_queue(QueueName=queue_name)['QueueUrl'] queue_arn = aws_stack.sqs_queue_arn(queue_name) delay_time = 1 testutil.create_lambda_function( func_name=func_name, zip_file=zip_file, handler=handler, runtime=runtime) lambda_client = aws_stack.connect_to_service('lambda') lambda_client.create_event_source_mapping( EventSourceArn=queue_arn, FunctionName=func_name ) self.client.send_message( QueueUrl=queue_url, MessageBody='hello world.', DelaySeconds=delay_time ) # assert that the Lambda has been invoked def get_logs(): logs = get_lambda_log_events(func_name) self.assertGreater(len(logs), 0) retry(get_logs, retries=5, sleep=3) # assert that the message has been deleted from the queue resp = self.client.receive_message(QueueUrl=queue_url, MessageAttributeNames=['All']) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) self.assertEqual(None, resp.get('Messages', None)) # clean up self.client.delete_queue(QueueUrl=queue_url) testutil.delete_lambda_function(func_name)
def __run_test(self, func_name, zip_file, handler, runtime, expected_lines): if not use_docker(): return testutil.create_lambda_function(func_name=func_name, zip_file=zip_file, handler=handler, runtime=runtime) result = self.lambda_client.invoke(FunctionName=func_name, Payload=b'{}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertEqual(to_str(result_data).strip(), '{}') # TODO make lambda log checks more resilient to various formats # self.check_lambda_logs(func_name, expected_lines=expected_lines) testutil.delete_lambda_function(func_name)
def test_python3_runtime_multple_create_with_conflicting_module(self): original_do_use_docker = lambda_api.DO_USE_DOCKER try: # always use the local runner lambda_api.DO_USE_DOCKER = False python3_with_settings1 = load_file( TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE1, mode='rb') python3_with_settings2 = load_file( TEST_LAMBDA_PYTHON3_MULTIPLE_CREATE2, mode='rb') lambda_name1 = 'test1-%s' % short_uid() testutil.create_lambda_function(func_name=lambda_name1, zip_file=python3_with_settings1, runtime=LAMBDA_RUNTIME_PYTHON36, handler='handler1.handler') lambda_name2 = 'test2-%s' % short_uid() testutil.create_lambda_function(func_name=lambda_name2, zip_file=python3_with_settings2, runtime=LAMBDA_RUNTIME_PYTHON36, handler='handler2.handler') result1 = self.lambda_client.invoke(FunctionName=lambda_name1, Payload=b'{}') result_data1 = result1['Payload'].read() result2 = self.lambda_client.invoke(FunctionName=lambda_name2, Payload=b'{}') result_data2 = result2['Payload'].read() self.assertEqual(result1['StatusCode'], 200) self.assertIn('setting1', to_str(result_data1)) self.assertEqual(result2['StatusCode'], 200) self.assertIn('setting2', to_str(result_data2)) # clean up testutil.delete_lambda_function(lambda_name1) testutil.delete_lambda_function(lambda_name2) finally: lambda_api.DO_USE_DOCKER = original_do_use_docker
def test_upload_lambda_from_s3(self): lambda_name = 'test_lambda_%s' % short_uid() bucket_name = 'test_bucket_lambda' bucket_key = 'test_lambda.zip' # upload zip file to S3 zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) self.s3_client.create_bucket(Bucket=bucket_name) self.s3_client.upload_fileobj(BytesIO(zip_file), bucket_name, bucket_key) # create lambda function self.lambda_client.create_function( FunctionName=lambda_name, Handler='handler.handler', Runtime=lambda_api.LAMBDA_RUNTIME_PYTHON27, Role='r1', Code={ 'S3Bucket': bucket_name, 'S3Key': bucket_key }) # invoke lambda function data_before = b'{"foo": "bar with \'quotes\\""}' result = self.lambda_client.invoke(FunctionName=lambda_name, Payload=data_before) data_after = json.loads(result['Payload'].read()) self.assertEqual(json.loads(to_str(data_before)), data_after['event']) context = data_after['context'] self.assertEqual('$LATEST', context['function_version']) self.assertEqual(lambda_name, context['function_name']) # clean up testutil.delete_lambda_function(lambda_name)
def test_destroy_idle_containers(self): executor = lambda_api.LAMBDA_EXECUTOR func_name = "test_destroy_idle_containers" func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone executor.destroy_existing_docker_containers() self.assertEqual(0, len(executor.get_all_container_names())) # deploy and invoke lambda without Docker testutil.create_lambda_function( func_name=func_name, handler_file=TEST_LAMBDA_ENV, libs=TEST_LAMBDA_LIBS, envvars={"Hello": "World"}, ) self.assertEqual(0, len(executor.get_all_container_names())) self.lambda_client.invoke(FunctionName=func_name, Payload=b"{}") self.assertEqual(1, len(executor.get_all_container_names())) # try to destroy idle containers. executor.idle_container_destroyer() self.assertEqual(1, len(executor.get_all_container_names())) # simulate an idle container executor.function_invoke_times[func_arn] = ( int(time.time() * 1000) - lambda_executors.MAX_CONTAINER_IDLE_TIME_MS ) executor.idle_container_destroyer() def assert_container_destroyed(): self.assertEqual(0, len(executor.get_all_container_names())) retry(assert_container_destroyed, retries=3) # clean up testutil.delete_lambda_function(func_name)
def test_lambda_put_item_to_dynamodb(self): table_name = 'ddb-table-{}'.format(short_uid()) function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX, short_uid()) aws_stack.create_dynamodb_table(table_name, partition_key='id') testutil.create_lambda_function(handler_file=TEST_LAMBDA_PUT_ITEM_FILE, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36) data = {short_uid(): 'data-{}'.format(i) for i in range(3)} event = { 'table_name': table_name, 'region_name': config.DEFAULT_REGION, 'items': [{ 'id': k, 'data': v } for k, v in data.items()] } self.lambda_client.invoke(FunctionName=function_name, Payload=json.dumps(event)) dynamodb = aws_stack.connect_to_resource('dynamodb') rs = dynamodb.Table(table_name).scan() items = rs['Items'] self.assertEqual(len(items), len(data.keys())) for item in items: self.assertEqual(data[item['id']], item['data']) # clean up testutil.delete_lambda_function(function_name) dynamodb_client = aws_stack.connect_to_service('dynamodb') dynamodb_client.delete_table(TableName=table_name)
def test_dotnet_lambda_running_in_docker(self): if not use_docker(): return testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_DOTNETCORE2, zip_file=self.zip_file_content, handler='DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler', runtime=LAMBDA_RUNTIME_DOTNETCORE2 ) result = self.lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}') result_data = result['Payload'].read() self.assertEqual(result['StatusCode'], 200) self.assertEqual(to_str(result_data).strip(), '{}') # assert that logs are present expected = ['Running .NET Core 2.0 Lambda'] self.check_lambda_logs(TEST_LAMBDA_NAME_DOTNETCORE2, expected_lines=expected) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_DOTNETCORE2)
def test_lambda_send_message_to_sqs(self): function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX, short_uid()) queue_name = 'lambda-queue-{}'.format(short_uid()) sqs_client = aws_stack.connect_to_service('sqs') testutil.create_lambda_function( handler_file=TEST_LAMBDA_SEND_MESSAGE_FILE, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36) queue_url = sqs_client.create_queue(QueueName=queue_name)['QueueUrl'] event = { 'message': 'message-from-test-lambda-{}'.format(short_uid()), 'queue_name': queue_name, 'region_name': config.DEFAULT_REGION } self.lambda_client.invoke(FunctionName=function_name, Payload=json.dumps(event)) # assert that message has been received on the Queue def receive_message(): rs = sqs_client.receive_message(QueueUrl=queue_url, MessageAttributeNames=['All']) self.assertGreater(len(rs['Messages']), 0) return rs['Messages'][0] message = retry(receive_message, retries=3, sleep=2) self.assertEqual(message['Body'], event['message']) # clean up testutil.delete_lambda_function(function_name) sqs_client.delete_queue(QueueUrl=queue_url)
def setup_and_tear_down(): zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_ENV), get_content=True) zip_file2 = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON_ECHO), get_content=True) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_1, zip_file=zip_file, envvars={"Hello": TEST_RESULT_VALUE}, ) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_2, zip_file=zip_file, envvars={"Hello": TEST_RESULT_VALUE}, ) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_3, zip_file=zip_file, envvars={"Hello": "Replace Value"}, ) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_4, zip_file=zip_file, envvars={"Hello": TEST_RESULT_VALUE}, ) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_5, zip_file=zip_file2) yield testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_1) testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_2) testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_3) testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_4) testutil.delete_lambda_function(name=TEST_LAMBDA_NAME_5)
def test_lambda_streams_batch_and_transactions(self): ddb_lease_table_suffix = '-kclapp2' table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME dynamodb = aws_stack.connect_to_service('dynamodb', client=True) dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(stream_name, delete=True) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == table_name: ddb_event_source_arn = stream['StreamArn'] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True) # submit a batch with writes dynamodb.batch_write_item( RequestItems={ table_name: [{ 'PutRequest': { 'Item': { PARTITION_KEY: { 'S': 'testId0' }, 'data': { 'S': 'foobar123' } } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: { 'S': 'testId1' }, 'data': { 'S': 'foobar123' } } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: { 'S': 'testId2' }, 'data': { 'S': 'foobar123' } } } }] }) # submit a batch with writes and deletes dynamodb.batch_write_item( RequestItems={ table_name: [ { 'PutRequest': { 'Item': { PARTITION_KEY: { 'S': 'testId3' }, 'data': { 'S': 'foobar123' } } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: { 'S': 'testId4' }, 'data': { 'S': 'foobar123' } } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: { 'S': 'testId5' }, 'data': { 'S': 'foobar123' } } } }, { 'DeleteRequest': { 'Key': { PARTITION_KEY: { 'S': 'testId0' } } } }, { 'DeleteRequest': { 'Key': { PARTITION_KEY: { 'S': 'testId1' } } } }, { 'DeleteRequest': { 'Key': { PARTITION_KEY: { 'S': 'testId2' } } } }, ] }) # submit a transaction with writes and delete dynamodb.transact_write_items(TransactItems=[ { 'Put': { 'TableName': table_name, 'Item': { PARTITION_KEY: { 'S': 'testId6' }, 'data': { 'S': 'foobar123' } } } }, { 'Put': { 'TableName': table_name, 'Item': { PARTITION_KEY: { 'S': 'testId7' }, 'data': { 'S': 'foobar123' } } } }, { 'Put': { 'TableName': table_name, 'Item': { PARTITION_KEY: { 'S': 'testId8' }, 'data': { 'S': 'foobar123' } } } }, { 'Delete': { 'TableName': table_name, 'Key': { PARTITION_KEY: { 'S': 'testId3' } } } }, { 'Delete': { 'TableName': table_name, 'Key': { PARTITION_KEY: { 'S': 'testId4' } } } }, { 'Delete': { 'TableName': table_name, 'Key': { PARTITION_KEY: { 'S': 'testId5' } } } }, ]) # submit a batch with a put over existing item dynamodb.transact_write_items(TransactItems=[ { 'Put': { 'TableName': table_name, 'Item': { PARTITION_KEY: { 'S': 'testId6' }, 'data': { 'S': 'foobar123_updated1' } } } }, ]) # submit a transaction with a put over existing item dynamodb.transact_write_items(TransactItems=[ { 'Put': { 'TableName': table_name, 'Item': { PARTITION_KEY: { 'S': 'testId7' }, 'data': { 'S': 'foobar123_updated1' } } } }, ]) # submit a transaction with updates dynamodb.transact_write_items(TransactItems=[ { 'Update': { 'TableName': table_name, 'Key': { PARTITION_KEY: { 'S': 'testId6' } }, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': { '#0': 'data' }, 'ExpressionAttributeValues': { ':0': { 'S': 'foobar123_updated2' } } } }, { 'Update': { 'TableName': table_name, 'Key': { PARTITION_KEY: { 'S': 'testId7' } }, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': { '#0': 'data' }, 'ExpressionAttributeValues': { ':0': { 'S': 'foobar123_updated2' } } } }, { 'Update': { 'TableName': table_name, 'Key': { PARTITION_KEY: { 'S': 'testId8' } }, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': { '#0': 'data' }, 'ExpressionAttributeValues': { ':0': { 'S': 'foobar123_updated2' } } } }, ]) LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_insert = 9 num_modify = 5 num_delete = 6 num_events = num_insert + num_modify + num_delete def check_events(): if len(events) != num_events: LOGGER.warning( ('DynamoDB updates retrieved (actual/expected): %s/%s') % (len(events), num_events)) self.assertEqual(len(events), num_events) event_items = [ json.loads(base64.b64decode(e['data'])) for e in events ] # make sure the we have the right amount of expected event types inserts = [ e for e in event_items if e.get('__action_type') == 'INSERT' ] modifies = [ e for e in event_items if e.get('__action_type') == 'MODIFY' ] removes = [ e for e in event_items if e.get('__action_type') == 'REMOVE' ] self.assertEqual(len(inserts), num_insert) self.assertEqual(len(modifies), num_modify) self.assertEqual(len(removes), num_delete) for i, event in enumerate(inserts): self.assertNotIn('old_image', event) self.assertEqual(inserts[i]['new_image'], { 'id': 'testId%d' % i, 'data': 'foobar123' }) self.assertEqual(modifies[0]['old_image'], { 'id': 'testId6', 'data': 'foobar123' }) self.assertEqual(modifies[0]['new_image'], { 'id': 'testId6', 'data': 'foobar123_updated1' }) self.assertEqual(modifies[1]['old_image'], { 'id': 'testId7', 'data': 'foobar123' }) self.assertEqual(modifies[1]['new_image'], { 'id': 'testId7', 'data': 'foobar123_updated1' }) self.assertEqual(modifies[2]['old_image'], { 'id': 'testId6', 'data': 'foobar123_updated1' }) self.assertEqual(modifies[2]['new_image'], { 'id': 'testId6', 'data': 'foobar123_updated2' }) self.assertEqual(modifies[3]['old_image'], { 'id': 'testId7', 'data': 'foobar123_updated1' }) self.assertEqual(modifies[3]['new_image'], { 'id': 'testId7', 'data': 'foobar123_updated2' }) self.assertEqual(modifies[4]['old_image'], { 'id': 'testId8', 'data': 'foobar123' }) self.assertEqual(modifies[4]['new_image'], { 'id': 'testId8', 'data': 'foobar123_updated2' }) for i, event in enumerate(removes): self.assertEqual(event['old_image'], { 'id': 'testId%d' % i, 'data': 'foobar123' }) self.assertNotIn('new_image', event) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=3) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_kinesis_lambda_sns_ddb_sqs_streams(self): ddb_lease_table_suffix = '-kclapp' table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') sqs = aws_stack.connect_to_service('sqs') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=stream_name + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(stream_name, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == table_name: ddb_event_source_arn = stream['StreamArn'] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to SQS queue sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_QUEUE, zip_file=zip_file, event_source_arn=sqs_queue_info['QueueArn'], runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(table_name) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item( RequestItems={ table_name: [{ 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 £' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢' } } }] }) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item( TableName=table_name, Key={PARTITION_KEY: { 'S': 'testId%s' % i }}, AttributeUpdates={ 'data': { 'Action': 'PUT', 'Value': { 'S': 'foobar123_updated' } } }) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test messages num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe( TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) self.assertEqual(len(latest), 10) # send messages to SQS queue num_events_sqs = 4 for i in range(num_events_sqs): sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'], MessageBody=str(i)) LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs num_events = num_events_lambda + num_events_kinesis def check_events(): if len(events) != num_events: LOGGER.warning(( 'DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s' ) % (len(events), num_events)) self.assertEqual(len(events), num_events) event_items = [ json.loads(base64.b64decode(e['data'])) for e in events ] # make sure the we have the right amount of INSERT/MODIFY event types inserts = [ e for e in event_items if e.get('__action_type') == 'INSERT' ] modifies = [ e for e in event_items if e.get('__action_type') == 'MODIFY' ] self.assertEqual(len(inserts), num_put_new_items + num_batch_items) self.assertEqual(len(modifies), num_put_existing_items + num_updates_ddb) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=3) # check cloudwatch notifications num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM) # TODO: It seems that CloudWatch is currently reporting an incorrect number of # invocations, namely the sum over *all* lambdas, not the single one we're asking for. # Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single # Lambda invocation may happen with a set of Kinesis records, hence we cannot simply # add num_events_ddb to num_events_lambda above! # self.assertEqual(num_invocations, 2 + num_events_lambda) self.assertGreater(num_invocations, num_events_sns + num_events_sqs) num_error_invocations = get_lambda_invocations_count( TEST_LAMBDA_NAME_STREAM, 'Errors') self.assertEqual(num_error_invocations, 1) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM) testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def test_sqs_batch_lambda_forward(self): sqs = aws_stack.connect_to_service('sqs') lambda_api = aws_stack.connect_to_service('lambda') lambda_name_queue_batch = 'lambda_queue_batch-%s' % short_uid() # deploy test lambda connected to SQS queue sqs_queue_info = testutil.create_sqs_queue(lambda_name_queue_batch) queue_url = sqs_queue_info['QueueUrl'] resp = testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON_ECHO, func_name=lambda_name_queue_batch, event_source_arn=sqs_queue_info['QueueArn'], runtime=LAMBDA_RUNTIME_PYTHON27, libs=TEST_LAMBDA_LIBS) event_source_id = resp['CreateEventSourceMappingResponse']['UUID'] lambda_api.update_event_source_mapping(UUID=event_source_id, BatchSize=5) messages_to_send = [{ 'Id': 'message{:02d}'.format(i), 'MessageBody': 'msgBody{:02d}'.format(i), 'MessageAttributes': { 'CustomAttribute': { 'DataType': 'String', 'StringValue': 'CustomAttributeValue{:02d}'.format(i) } } } for i in range(1, 12)] # send 11 messages (which should get split into 3 batches) sqs.send_message_batch(QueueUrl=queue_url, Entries=messages_to_send[:10]) sqs.send_message( QueueUrl=queue_url, MessageBody=messages_to_send[10]['MessageBody'], MessageAttributes=messages_to_send[10]['MessageAttributes']) def wait_for_done(): attributes = sqs.get_queue_attributes( QueueUrl=queue_url, AttributeNames=[ 'ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesDelayed', 'ApproximateNumberOfMessagesNotVisible' ], )['Attributes'] msg_count = int(attributes.get('ApproximateNumberOfMessages')) self.assertEqual(msg_count, 0, 'expecting queue to be empty') delayed_count = int( attributes.get('ApproximateNumberOfMessagesDelayed')) if delayed_count != 0: LOGGER.warning( 'SQS delayed message count (actual/expected): %s/%s' % (delayed_count, 0)) not_visible_count = int( attributes.get('ApproximateNumberOfMessagesNotVisible')) if not_visible_count != 0: LOGGER.warning( 'SQS messages not visible (actual/expected): %s/%s' % (not_visible_count, 0)) self.assertEqual(delayed_count, 0, 'no messages waiting for retry') self.assertEqual(delayed_count + not_visible_count, 0, 'no in flight messages') # wait for the queue to drain (max 60s) retry(wait_for_done, retries=12, sleep=5.0) events = get_lambda_log_events(lambda_name_queue_batch, 10) self.assertEqual(len(events), 3, 'expected 3 lambda invocations') testutil.delete_lambda_function(lambda_name_queue_batch) sqs.delete_queue(QueueUrl=queue_url)
def tearDownClass(cls): testutil.delete_lambda_function(cls.scheduled_lambda_name)
def test_lambda_streams_batch_and_transactions(self): ddb_lease_table_suffix = "-kclapp2" table_name = TEST_TABLE_NAME + "lsbat" + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME lambda_ddb_name = "lambda-ddb-%s" % short_uid() dynamodb = aws_stack.create_external_boto_client("dynamodb", client=True) dynamodb_service = aws_stack.create_external_boto_client("dynamodb") dynamodbstreams = aws_stack.create_external_boto_client( "dynamodbstreams") LOGGER.info("Creating test streams...") run_safe( lambda: dynamodb_service.delete_table(TableName=stream_name + ddb_lease_table_suffix), print_error=False, ) aws_stack.create_kinesis_stream(stream_name, delete=True) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix, ) LOGGER.info("Kinesis consumer initialized.") # create table with stream forwarding config aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type="NEW_AND_OLD_IMAGES", ) # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams["Streams"]: if stream["TableName"] == table_name: ddb_event_source_arn = stream["StreamArn"] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream testutil.create_lambda_function( handler_file=TEST_LAMBDA_PYTHON, libs=TEST_LAMBDA_LIBS, func_name=lambda_ddb_name, event_source_arn=ddb_event_source_arn, delete=True, ) # submit a batch with writes dynamodb.batch_write_item( RequestItems={ table_name: [ { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId0" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId1" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId2" }, "data": { "S": "foobar123" }, } } }, ] }) # submit a batch with writes and deletes dynamodb.batch_write_item( RequestItems={ table_name: [ { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId3" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId4" }, "data": { "S": "foobar123" }, } } }, { "PutRequest": { "Item": { PARTITION_KEY: { "S": "testId5" }, "data": { "S": "foobar123" }, } } }, { "DeleteRequest": { "Key": { PARTITION_KEY: { "S": "testId0" } } } }, { "DeleteRequest": { "Key": { PARTITION_KEY: { "S": "testId1" } } } }, { "DeleteRequest": { "Key": { PARTITION_KEY: { "S": "testId2" } } } }, ] }) # submit a transaction with writes and delete dynamodb.transact_write_items(TransactItems=[ { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId6" }, "data": { "S": "foobar123" }, }, } }, { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId7" }, "data": { "S": "foobar123" }, }, } }, { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId8" }, "data": { "S": "foobar123" }, }, } }, { "Delete": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId3" } }, } }, { "Delete": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId4" } }, } }, { "Delete": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId5" } }, } }, ]) # submit a batch with a put over existing item dynamodb.transact_write_items(TransactItems=[ { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId6" }, "data": { "S": "foobar123_updated1" }, }, } }, ]) # submit a transaction with a put over existing item dynamodb.transact_write_items(TransactItems=[ { "Put": { "TableName": table_name, "Item": { PARTITION_KEY: { "S": "testId7" }, "data": { "S": "foobar123_updated1" }, }, } }, ]) # submit a transaction with updates dynamodb.transact_write_items(TransactItems=[ { "Update": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId6" } }, "UpdateExpression": "SET #0 = :0", "ExpressionAttributeNames": { "#0": "data" }, "ExpressionAttributeValues": { ":0": { "S": "foobar123_updated2" } }, } }, { "Update": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId7" } }, "UpdateExpression": "SET #0 = :0", "ExpressionAttributeNames": { "#0": "data" }, "ExpressionAttributeValues": { ":0": { "S": "foobar123_updated2" } }, } }, { "Update": { "TableName": table_name, "Key": { PARTITION_KEY: { "S": "testId8" } }, "UpdateExpression": "SET #0 = :0", "ExpressionAttributeNames": { "#0": "data" }, "ExpressionAttributeValues": { ":0": { "S": "foobar123_updated2" } }, } }, ]) LOGGER.info("Waiting some time before finishing test.") time.sleep(2) num_insert = 9 num_modify = 5 num_delete = 6 num_events = num_insert + num_modify + num_delete def check_events(): if len(events) != num_events: msg = "DynamoDB updates retrieved (actual/expected): %s/%s" % ( len(events), num_events, ) LOGGER.warning(msg) self.assertEqual(num_events, len(events)) event_items = [ json.loads(base64.b64decode(e["data"])) for e in events ] # make sure the we have the right amount of expected event types inserts = [ e for e in event_items if e.get("__action_type") == "INSERT" ] modifies = [ e for e in event_items if e.get("__action_type") == "MODIFY" ] removes = [ e for e in event_items if e.get("__action_type") == "REMOVE" ] self.assertEqual(num_insert, len(inserts)) self.assertEqual(num_modify, len(modifies)) self.assertEqual(num_delete, len(removes)) # assert that all inserts were received for i, event in enumerate(inserts): self.assertNotIn("old_image", event) item_id = "testId%d" % i matching = [ i for i in inserts if i["new_image"]["id"] == item_id ][0] self.assertEqual({ "id": item_id, "data": "foobar123" }, matching["new_image"]) # assert that all updates were received def assert_updates(expected_updates, modifies): def found(update): for modif in modifies: if modif["old_image"]["id"] == update["id"]: self.assertEqual( modif["old_image"], { "id": update["id"], "data": update["old"] }, ) self.assertEqual( modif["new_image"], { "id": update["id"], "data": update["new"] }, ) return True for update in expected_updates: self.assertTrue(found(update)) updates1 = [ { "id": "testId6", "old": "foobar123", "new": "foobar123_updated1" }, { "id": "testId7", "old": "foobar123", "new": "foobar123_updated1" }, ] updates2 = [ { "id": "testId6", "old": "foobar123_updated1", "new": "foobar123_updated2", }, { "id": "testId7", "old": "foobar123_updated1", "new": "foobar123_updated2", }, { "id": "testId8", "old": "foobar123", "new": "foobar123_updated2" }, ] assert_updates(updates1, modifies[:2]) assert_updates(updates2, modifies[2:]) # assert that all removes were received for i, event in enumerate(removes): self.assertNotIn("new_image", event) item_id = "testId%d" % i matching = [ i for i in removes if i["old_image"]["id"] == item_id ][0] self.assertEqual({ "id": item_id, "data": "foobar123" }, matching["old_image"]) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=4) # clean up testutil.delete_lambda_function(lambda_ddb_name)