def test_upload_lambda_from_s3(): s3_client = aws_stack.connect_to_service('s3') lambda_client = aws_stack.connect_to_service('lambda') lambda_name = 'test_lambda_%s' % short_uid() bucket_name = 'test_bucket_lambda' bucket_key = 'test_lambda.zip' # upload zip file to S3 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) s3_client.create_bucket(Bucket=bucket_name) s3_client.upload_fileobj(BytesIO(zip_file), bucket_name, bucket_key) # create lambda function lambda_client.create_function( FunctionName=lambda_name, Handler='handler.handler', Runtime=lambda_api.LAMBDA_RUNTIME_PYTHON27, Role='r1', Code={ 'S3Bucket': bucket_name, 'S3Key': bucket_key } ) # invoke lambda function data_before = b'{"foo": "bar"}' result = lambda_client.invoke(FunctionName=lambda_name, Payload=data_before) data_after = result['Payload'].read() assert json.loads(to_str(data_before)) == json.loads(to_str(data_after))
def test_kinesis_lambda_forward_chain(): kinesis = aws_stack.connect_to_service('kinesis') s3 = aws_stack.connect_to_service('s3') aws_stack.create_kinesis_stream(TEST_CHAIN_STREAM1_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_CHAIN_STREAM2_NAME, delete=True) s3.create_bucket(Bucket=TEST_BUCKET_NAME) # deploy test lambdas connected to Kinesis streams zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_CHAIN_LAMBDA1_NAME, zip_file=zip_file, event_source_arn=get_event_source_arn(TEST_CHAIN_STREAM1_NAME), runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_CHAIN_LAMBDA2_NAME, zip_file=zip_file, event_source_arn=get_event_source_arn(TEST_CHAIN_STREAM2_NAME), runtime=LAMBDA_RUNTIME_PYTHON27) # publish test record test_data = {'test_data': 'forward_chain_data_%s' % short_uid()} data = clone(test_data) data[lambda_integration.MSG_BODY_MESSAGE_TARGET] = 'kinesis:%s' % TEST_CHAIN_STREAM2_NAME kinesis.put_record(Data=to_bytes(json.dumps(data)), PartitionKey='testId', StreamName=TEST_CHAIN_STREAM1_NAME) # check results time.sleep(5) all_objects = testutil.list_all_s3_objects() testutil.assert_objects(test_data, all_objects)
def get_machine_id(): global MACHINE_ID if MACHINE_ID: return MACHINE_ID # determine MACHINE_ID from config files configs_map = {} config_file_tmp = get_config_file_tempdir() config_file_home = get_config_file_homedir() for config_file in (config_file_home, config_file_tmp): if config_file: local_configs = load_file(config_file) local_configs = json.loads(to_str(local_configs)) configs_map[config_file] = local_configs if 'machine_id' in local_configs: MACHINE_ID = local_configs['machine_id'] break # if we can neither find NOR create the config files, fall back to process id if not configs_map: return PROCESS_ID # assign default id if empty if not MACHINE_ID: MACHINE_ID = short_uid() # update MACHINE_ID in all config files for config_file, configs in configs_map.items(): configs['machine_id'] = MACHINE_ID save_file(config_file, json.dumps(configs)) return MACHINE_ID
def get_function_code(function): """ Get the code of an existing function --- operationId: 'getFunctionCode' parameters: """ arn = func_arn(function) lambda_cwd = arn_to_lambda[arn].cwd tmp_file = '%s/%s' % (lambda_cwd, LAMBDA_ZIP_FILE_NAME) return Response(load_file(tmp_file, mode='rb'), mimetype='application/zip', headers={'Content-Disposition': 'attachment; filename=lambda_archive.zip'})
def test_lambda_environment(): lambda_client = aws_stack.connect_to_service('lambda') # deploy and invoke lambda without Docker zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_ENV), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_ENV, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'}) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_ENV, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'] assert json.load(result_data) == {'Hello': 'World'}
def test_list_stack_resources_returns_queue_urls(self): cloudformation = aws_stack.connect_to_resource('cloudformation') template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_2)) cloudformation.create_stack(StackName=TEST_STACK_NAME_2, TemplateBody=template) def check_stack(): stack = get_stack_details(TEST_STACK_NAME_2) assert stack['StackStatus'] == 'CREATE_COMPLETE' retry(check_stack, retries=3, sleep=2) list_stack_summaries = list_stack_resources(TEST_STACK_NAME_2) queue_urls = get_queue_urls() for resource in list_stack_summaries: assert resource['PhysicalResourceId'] in queue_urls
def get_lambda_code(func_name, retries=1, cache_time=None, env=None): if MOCK_OBJ: return '' env = aws_stack.get_environment(env) if cache_time is None and env.region != REGION_LOCAL: cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT out = cmd_lambda('get-function --function-name %s' % func_name, env, cache_time) out = json.loads(out) loc = out['Code']['Location'] hash = md5(loc) folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash) filename = 'archive.zip' archive = '%s/%s' % (folder, filename) try: mkdir(folder) if not os.path.isfile(archive): download(loc, archive, verify_ssl=False) if len(os.listdir(folder)) <= 1: zip_path = os.path.join(folder, filename) unzip(zip_path, folder) except Exception as e: print('WARN: %s' % e) rm_rf(archive) if retries > 0: return get_lambda_code(func_name, retries=retries - 1, cache_time=1, env=env) else: print('WARNING: Unable to retrieve lambda code: %s' % e) # traverse subdirectories and get script sources result = {} for root, subdirs, files in os.walk(folder): for file in files: prefix = root.split(folder)[-1] key = '%s/%s' % (prefix, file) if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key): codefile = '%s/%s' % (root, file) result[key] = load_file(codefile) # cleanup cache clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN, last_clean_time=last_cache_cleanup_time, max_age=TMP_DOWNLOAD_CACHE_MAX_AGE) # TODO: delete only if cache_time is over rm_rf(folder) return result
def test_apply_template(self): cloudformation = aws_stack.connect_to_resource('cloudformation') template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1)) # deploy template cloudformation.create_stack(StackName=TEST_STACK_NAME, TemplateBody=template) # wait for deployment to finish def check_stack(): stack = get_stack_details(TEST_STACK_NAME) assert stack['StackStatus'] == 'CREATE_COMPLETE' retry(check_stack, retries=3, sleep=2) # assert that bucket has been created assert bucket_exists('cf-test-bucket-1') # assert that queue has been created assert queue_exists('cf-test-queue-1') # assert that stream has been created assert stream_exists('cf-test-stream-1') # assert that queue has been created resource = describe_stack_resource(TEST_STACK_NAME, 'SQSQueueNoNameProperty') assert queue_exists(resource['PhysicalResourceId'])
def test_destroy_idle_containers(): # run these tests only for the "reuse containers" Lambda executor if not isinstance(lambda_api.LAMBDA_EXECUTOR, lambda_executors.LambdaExecutorReuseContainers): return executor = lambda_api.LAMBDA_EXECUTOR func_name = 'test_destroy_idle_containers' # create a new lambda lambda_client = aws_stack.connect_to_service('lambda') func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone executor.destroy_existing_docker_containers() assert len(executor.get_all_container_names()) == 0 # deploy and invoke lambda without Docker zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_ENV), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=func_name, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'}) assert len(executor.get_all_container_names()) == 0 lambda_client.invoke(FunctionName=func_name, Payload=b'{}') assert len(executor.get_all_container_names()) == 1 # try to destroy idle containers. executor.idle_container_destroyer() assert len(executor.get_all_container_names()) == 1 # simulate an idle container executor.function_invoke_times[func_arn] = time.time() - 610 executor.idle_container_destroyer() assert len(executor.get_all_container_names()) == 0
def test_api_gateway_lambda_proxy_integration(): # create lambda function zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_PROXY_BACKEND, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27) # create API Gateway and connect it to the Lambda proxy backend lambda_uri = aws_stack.lambda_function_arn(TEST_LAMBDA_PROXY_BACKEND) target_uri = 'arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations' % (DEFAULT_REGION, lambda_uri) result = connect_api_gateway_to_http_with_lambda_proxy('test_gateway2', target_uri, path=API_PATH_LAMBDA_PROXY_BACKEND) # make test request to gateway and check response path = API_PATH_LAMBDA_PROXY_BACKEND.replace('{test_param1}', 'foo1') url = INBOUND_GATEWAY_URL_PATTERN.format(api_id=result['id'], stage_name=TEST_STAGE_NAME, path=path) data = {'return_status_code': 203, 'return_headers': {'foo': 'bar123'}} result = requests.post(url, data=json.dumps(data)) assert result.status_code == 203 assert result.headers.get('foo') == 'bar123' parsed_body = json.loads(to_str(result.content)) assert parsed_body.get('return_status_code') == 203 assert parsed_body.get('return_headers') == {'foo': 'bar123'} assert parsed_body.get('pathParameters') == {'test_param1': 'foo1'}
def test_kinesis_lambda_sns_ddb_sqs_streams(self): ddb_lease_table_suffix = '-kclapp' table_name = TEST_TABLE_NAME + 'klsdss' + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') sqs = aws_stack.connect_to_service('sqs') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=stream_name + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(stream_name, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == table_name: ddb_event_source_arn = stream['StreamArn'] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to SQS queue sqs_queue_info = testutil.create_sqs_queue(TEST_LAMBDA_NAME_QUEUE) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_QUEUE, zip_file=zip_file, event_source_arn=sqs_queue_info['QueueArn'], runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(table_name) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item(RequestItems={table_name: [ {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}} ]}) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item(TableName=table_name, Key={PARTITION_KEY: {'S': 'testId%s' % i}}, AttributeUpdates={'data': { 'Action': 'PUT', 'Value': {'S': 'foobar123_updated'} }}) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test messages num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) self.assertEqual(len(latest), 10) # send messages to SQS queue num_events_sqs = 4 for i in range(num_events_sqs): sqs.send_message(QueueUrl=sqs_queue_info['QueueUrl'], MessageBody=str(i)) LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs num_events = num_events_lambda + num_events_kinesis def check_events(): if len(events) != num_events: LOGGER.warning(('DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s') % (len(events), num_events)) self.assertEqual(len(events), num_events) event_items = [json.loads(base64.b64decode(e['data'])) for e in events] # make sure the we have the right amount of INSERT/MODIFY event types inserts = [e for e in event_items if e.get('__action_type') == 'INSERT'] modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY'] self.assertEqual(len(inserts), num_put_new_items + num_batch_items) self.assertEqual(len(modifies), num_put_existing_items + num_updates_ddb) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=3) # check cloudwatch notifications num_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM) # TODO: It seems that CloudWatch is currently reporting an incorrect number of # invocations, namely the sum over *all* lambdas, not the single one we're asking for. # Also, we need to bear in mind that Kinesis may perform batch updates, i.e., a single # Lambda invocation may happen with a set of Kinesis records, hence we cannot simply # add num_events_ddb to num_events_lambda above! # self.assertEqual(num_invocations, 2 + num_events_lambda) self.assertGreater(num_invocations, num_events_sns + num_events_sqs) num_error_invocations = get_lambda_invocations_count(TEST_LAMBDA_NAME_STREAM, 'Errors') self.assertEqual(num_error_invocations, 1) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_STREAM) testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def set_function_code(code, lambda_name, lambda_cwd=None): def generic_handler(event, context): raise ClientError(( 'Unable to find executor for Lambda function "%s". Note that ' + 'Node.js, Golang, and .Net Core Lambdas currently require LAMBDA_EXECUTOR=docker' ) % lambda_name) arn = func_arn(lambda_name) lambda_details = arn_to_lambda[arn] runtime = lambda_details.runtime lambda_environment = lambda_details.envvars handler_name = lambda_details.handler or LAMBDA_DEFAULT_HANDLER code_passed = code code = code or lambda_details.code is_local_mount = code.get('S3Bucket') == BUCKET_MARKER_LOCAL zip_file_content = None if code_passed: lambda_cwd = lambda_cwd or set_archive_code(code_passed, lambda_name) if not is_local_mount: # Save the zip file to a temporary file that the lambda executors can reference zip_file_content = get_zip_bytes(code_passed) else: lambda_cwd = lambda_cwd or lambda_details.cwd # get local lambda working directory tmp_file = '%s/%s' % (lambda_cwd, LAMBDA_ZIP_FILE_NAME) if not zip_file_content: zip_file_content = load_file(tmp_file, mode='rb') # Set the appropriate lambda handler. lambda_handler = generic_handler if runtime == LAMBDA_RUNTIME_JAVA8: # The Lambda executors for Docker subclass LambdaExecutorContainers, # which runs Lambda in Docker by passing all *.jar files in the function # working directory as part of the classpath. Because of this, we need to # save the zip_file_content as a .jar here. lambda_handler, zip_file_content = get_java_handler( zip_file_content, handler_name, tmp_file) if is_jar_archive(zip_file_content): jar_tmp_file = '{working_dir}/{file_name}'.format( working_dir=lambda_cwd, file_name=LAMBDA_JAR_FILE_NAME) save_file(jar_tmp_file, zip_file_content) else: handler_file = get_handler_file_from_name(handler_name, runtime=runtime) handler_function = get_handler_function_from_name(handler_name, runtime=runtime) if not is_local_mount: # Lambda code must be uploaded in Zip format if not is_zip_file(zip_file_content): raise ClientError( 'Uploaded Lambda code for runtime ({}) is not in Zip format' .format(runtime)) unzip(tmp_file, lambda_cwd) main_file = '%s/%s' % (lambda_cwd, handler_file) if not os.path.exists(main_file): # Raise an error if (1) this is not a local mount lambda, or (2) we're # running Lambdas locally (not in Docker), or (3) we're using remote Docker. # -> We do *not* want to raise an error if we're using local mount in non-remote Docker if not is_local_mount or not use_docker( ) or config.LAMBDA_REMOTE_DOCKER: file_list = run('cd "%s"; du -d 3 .' % lambda_cwd) config_debug = ( 'Config for local mount, docker, remote: "%s", "%s", "%s"' % (is_local_mount, use_docker(), config.LAMBDA_REMOTE_DOCKER)) LOG.debug('Lambda archive content:\n%s' % file_list) raise ClientError( error_response( 'Unable to find handler script in Lambda archive. %s' % config_debug, 400, error_type='ValidationError')) if runtime.startswith('python') and not use_docker(): try: # make sure the file is actually readable, then read contents ensure_readable(main_file) zip_file_content = load_file(main_file, mode='rb') # extract handler lambda_handler = exec_lambda_code( zip_file_content, handler_function=handler_function, lambda_cwd=lambda_cwd, lambda_env=lambda_environment) except Exception as e: raise ClientError( 'Unable to get handler function from lambda code.', e) add_function_mapping(lambda_name, lambda_handler, lambda_cwd) return {'FunctionName': lambda_name}
def install_elasticsearch(version=None): version = get_elasticsearch_install_version(version) install_dir = get_elasticsearch_install_dir(version) installed_executable = os.path.join(install_dir, 'bin', 'elasticsearch') if not os.path.exists(installed_executable): log_install_msg('Elasticsearch (%s)' % version) es_url = ELASTICSEARCH_URLS.get(version) if not es_url: raise Exception( 'Unable to find download URL for Elasticsearch version "%s"' % version) install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join(config.TMP_FOLDER, 'localstack.%s' % os.path.basename(es_url)) download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob( os.path.join(install_dir_parent, 'elasticsearch*')) if not elasticsearch_dir: raise Exception('Unable to find Elasticsearch folder in %s' % install_dir_parent) shutil.move(elasticsearch_dir[0], install_dir) for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'): dir_path = os.path.join(install_dir, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # install default plugins for plugin in ELASTICSEARCH_PLUGIN_LIST: if is_alpine(): # https://github.com/pires/docker-elasticsearch/issues/56 os.environ['ES_TMPDIR'] = '/tmp' plugin_binary = os.path.join(install_dir, 'bin', 'elasticsearch-plugin') plugin_dir = os.path.join(install_dir, 'plugins', plugin) if not os.path.exists(plugin_dir): LOG.info('Installing Elasticsearch plugin %s' % (plugin)) run('%s install -b %s' % (plugin_binary, plugin)) # delete some plugins to free up space for plugin in ELASTICSEARCH_DELETE_MODULES: module_dir = os.path.join(install_dir, 'modules', plugin) rm_rf(module_dir) # disable x-pack-ml plugin (not working on Alpine) xpack_dir = os.path.join(install_dir, 'modules', 'x-pack-ml', 'platform') rm_rf(xpack_dir) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(install_dir, 'config', 'jvm.options') if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)', r'# \1', jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def test_prime_and_destroy_containers(): # run these tests only for the "reuse containers" Lambda executor if not isinstance(lambda_api.LAMBDA_EXECUTOR, lambda_executors.LambdaExecutorReuseContainers): return executor = lambda_api.LAMBDA_EXECUTOR func_name = 'test_prime_and_destroy_containers' # create a new lambda lambda_client = aws_stack.connect_to_service('lambda') func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone executor.cleanup() assert len(executor.get_all_container_names()) == 0 # deploy and invoke lambda without Docker zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_ENV), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=func_name, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'}) assert len(executor.get_all_container_names()) == 0 assert executor.function_invoke_times == {} # invoke a few times. durations = [] num_iterations = 3 for i in range(0, num_iterations + 1): prev_invoke_time = None if i > 0: prev_invoke_time = executor.function_invoke_times[func_arn] start_time = time.time() lambda_client.invoke(FunctionName=func_name, Payload=b'{}') duration = time.time() - start_time assert len(executor.get_all_container_names()) == 1 # ensure the last invoke time is being updated properly. if i > 0: assert executor.function_invoke_times[func_arn] > prev_invoke_time else: assert executor.function_invoke_times[func_arn] > 0 durations.append(duration) # the first call would have created the container. subsequent calls would reuse and be faster. for i in range(1, num_iterations + 1): assert durations[i] < durations[0] status = executor.get_docker_container_status(func_arn) assert status == 1 executor.cleanup() status = executor.get_docker_container_status(func_arn) assert status == 0 assert len(executor.get_all_container_names()) == 0
def test_validate_template(self): cloudformation = aws_stack.connect_to_service('cloudformation') template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1)) response = cloudformation.validate_template(TemplateBody=template) assert response['ResponseMetadata']['HTTPStatusCode'] == 200
def test_lambda_runtimes(): lambda_client = aws_stack.connect_to_service('lambda') # deploy and invoke lambda - Python 2.7 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' if use_docker(): # deploy and invoke lambda - Python 3.6 zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON3), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java if not os.path.exists(TEST_LAMBDA_JAVA): mkdir(os.path.dirname(TEST_LAMBDA_JAVA)) download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA) zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' if use_docker(): # deploy and invoke lambda - Node.js zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}'
def create_lambda_function( func_name, zip_file=None, event_source_arn=None, handler_file=None, handler=None, starting_position=None, runtime=None, envvars={}, tags={}, libs=[], delete=False, layers=None, **kwargs, ): """Utility method to create a new function via the Lambda API""" starting_position = starting_position or LAMBDA_DEFAULT_STARTING_POSITION runtime = runtime or LAMBDA_DEFAULT_RUNTIME client = aws_stack.connect_to_service("lambda") # load zip file content if handler_file is specified if not zip_file and handler_file: file_content = load_file(handler_file) if os.path.exists( handler_file) else handler_file if libs or not handler: zip_file = create_lambda_archive( file_content, libs=libs, get_content=True, runtime=runtime or LAMBDA_DEFAULT_RUNTIME, ) else: zip_file = create_zip_file(handler_file, get_content=True) handler = handler or LAMBDA_DEFAULT_HANDLER if delete: try: # Delete function if one already exists client.delete_function(FunctionName=func_name) except Exception: pass lambda_code = {"ZipFile": zip_file} if len(zip_file) > MAX_LAMBDA_ARCHIVE_UPLOAD_SIZE: s3 = aws_stack.connect_to_service("s3") aws_stack.get_or_create_bucket(LAMBDA_ASSETS_BUCKET_NAME) asset_key = f"{short_uid()}.zip" s3.upload_fileobj(Fileobj=io.BytesIO(zip_file), Bucket=LAMBDA_ASSETS_BUCKET_NAME, Key=asset_key) lambda_code = { "S3Bucket": LAMBDA_ASSETS_BUCKET_NAME, "S3Key": asset_key } # create function additional_kwargs = kwargs kwargs = { "FunctionName": func_name, "Runtime": runtime, "Handler": handler, "Role": LAMBDA_TEST_ROLE, "Code": lambda_code, "Timeout": LAMBDA_TIMEOUT_SEC, "Environment": dict(Variables=envvars), "Tags": tags, } kwargs.update(additional_kwargs) if layers: kwargs["Layers"] = layers create_func_resp = client.create_function(**kwargs) resp = { "CreateFunctionResponse": create_func_resp, "CreateEventSourceMappingResponse": None, } # create event source mapping if event_source_arn: resp[ "CreateEventSourceMappingResponse"] = client.create_event_source_mapping( FunctionName=func_name, EventSourceArn=event_source_arn, StartingPosition=starting_position, ) return resp
def setUpClass(cls): cls.lambda_client = aws_stack.connect_to_service('lambda') cls.zip_file_content2 = load_file(TEST_LAMBDA_DOTNETCORE2, mode='rb') cls.zip_file_content31 = load_file(TEST_LAMBDA_DOTNETCORE31, mode='rb')
def test_prime_and_destroy_containers(self): # run these tests only for the "reuse containers" Lambda executor if not isinstance(lambda_api.LAMBDA_EXECUTOR, lambda_executors.LambdaExecutorReuseContainers): return executor = lambda_api.LAMBDA_EXECUTOR func_name = 'test_prime_and_destroy_containers' func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone executor.cleanup() self.assertEqual(len(executor.get_all_container_names()), 0) # deploy and invoke lambda without Docker zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_ENV), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27 ) testutil.create_lambda_function( func_name=func_name, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'} ) self.assertEqual(len(executor.get_all_container_names()), 0) self.assertDictEqual(executor.function_invoke_times, {}) # invoke a few times. durations = [] num_iterations = 3 for i in range(0, num_iterations + 1): prev_invoke_time = None if i > 0: prev_invoke_time = executor.function_invoke_times[func_arn] start_time = time.time() self.lambda_client.invoke(FunctionName=func_name, Payload=b'{}') duration = time.time() - start_time self.assertEqual(len(executor.get_all_container_names()), 1) # ensure the last invoke time is being updated properly. if i > 0: self.assertGreater(executor.function_invoke_times[func_arn], prev_invoke_time) else: self.assertGreater(executor.function_invoke_times[func_arn], 0) durations.append(duration) # the first call would have created the container. subsequent calls would reuse and be faster. for i in range(1, num_iterations + 1): self.assertLess(durations[i], durations[0]) status = executor.get_docker_container_status(func_arn) self.assertEqual(status, 1) container_network = executor.get_docker_container_network(func_arn) self.assertEqual(container_network, 'default') executor.cleanup() status = executor.get_docker_container_status(func_arn) self.assertEqual(status, 0) self.assertEqual(len(executor.get_all_container_names()), 0)
def test_create_delete_stack(self): cloudformation = aws_stack.connect_to_resource('cloudformation') cf_client = aws_stack.connect_to_service('cloudformation') s3 = aws_stack.connect_to_service('s3') sns = aws_stack.connect_to_service('sns') apigateway = aws_stack.connect_to_service('apigateway') template = template_deployer.template_to_json( load_file(TEST_TEMPLATE_1)) # deploy template stack_name = 'stack-%s' % short_uid() cloudformation.create_stack(StackName=stack_name, TemplateBody=template) # wait for deployment to finish def check_stack(): stack = get_stack_details(stack_name) self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE') retry(check_stack, retries=3, sleep=2) # assert that resources have been created assert bucket_exists('cf-test-bucket-1') assert queue_exists('cf-test-queue-1') topic_arn = topic_exists('%s-test-topic-1-1' % stack_name) assert topic_arn assert stream_exists('cf-test-stream-1') resource = describe_stack_resource(stack_name, 'SQSQueueNoNameProperty') assert queue_exists(resource['PhysicalResourceId']) # assert that tags have been created tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet'] self.assertEqual( tags, [{ 'Key': 'foobar', 'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1') }]) tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags'] self.assertEqual( tags, [{ 'Key': 'foo', 'Value': 'cf-test-bucket-1' }, { 'Key': 'bar', 'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1') }]) # assert that subscriptions have been created subs = sns.list_subscriptions()['Subscriptions'] subs = [ s for s in subs if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint'] ] self.assertEqual(len(subs), 1) self.assertIn( ':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name), subs[0]['TopicArn']) # assert that Gateway responses have been created test_api_name = 'test-api' api = [ a for a in apigateway.get_rest_apis()['items'] if a['name'] == test_api_name ][0] responses = apigateway.get_gateway_responses( restApiId=api['id'])['items'] self.assertEqual(len(responses), 2) types = [r['responseType'] for r in responses] self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX'])) # delete the stack cf_client.delete_stack(StackName=stack_name) # assert that resources have been deleted assert not bucket_exists('cf-test-bucket-1') assert not queue_exists('cf-test-queue-1') assert not topic_exists('%s-test-topic-1-1' % stack_name) retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def test_lambda_runtimes(): lambda_client = aws_stack.connect_to_service('lambda') # deploy and invoke lambda - Python 2.7 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' if use_docker(): # deploy and invoke lambda - Python 3.6 zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON3), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java if not os.path.exists(TEST_LAMBDA_JAVA): mkdir(os.path.dirname(TEST_LAMBDA_JAVA)) download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA) zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'LinkedHashMap' in to_str(result_data) # test SNSEvent result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'async': 'True'} # test KinesisEvent result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JAVA, Payload= b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}' ) assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'KinesisEvent' in to_str(result_data) # deploy and invoke lambda - Java with stream handler testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' if use_docker(): # deploy and invoke lambda - Node.js zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}'
def _test_api_gateway_lambda_proxy_integration(self, fn_name, path): # create lambda function zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27 ) testutil.create_lambda_function( func_name=fn_name, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27 ) # create API Gateway and connect it to the Lambda proxy backend lambda_uri = aws_stack.lambda_function_arn(fn_name) invocation_uri = 'arn:aws:apigateway:%s:lambda:path/2015-03-31/functions/%s/invocations' target_uri = invocation_uri % (DEFAULT_REGION, lambda_uri) result = self.connect_api_gateway_to_http_with_lambda_proxy( 'test_gateway2', target_uri, path=path ) api_id = result['id'] path_map = get_rest_api_paths(api_id) _, resource = get_resource_for_path('/lambda/foo1', path_map) # make test request to gateway and check response path = path.replace('{test_param1}', 'foo1') path = path + '?foo=foo&bar=bar&bar=baz' url = INBOUND_GATEWAY_URL_PATTERN.format( api_id=api_id, stage_name=self.TEST_STAGE_NAME, path=path ) data = {'return_status_code': 203, 'return_headers': {'foo': 'bar123'}} result = requests.post( url, data=json.dumps(data), headers={'User-Agent': 'python-requests/testing'} ) self.assertEqual(result.status_code, 203) self.assertEqual(result.headers.get('foo'), 'bar123') parsed_body = json.loads(to_str(result.content)) self.assertEqual(parsed_body.get('return_status_code'), 203) self.assertDictEqual(parsed_body.get('return_headers'), {'foo': 'bar123'}) self.assertDictEqual(parsed_body.get('queryStringParameters'), {'foo': 'foo', 'bar': ['bar', 'baz']}) request_context = parsed_body.get('requestContext') source_ip = request_context['identity'].pop('sourceIp') self.assertTrue(re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', source_ip)) self.assertEqual(request_context['path'], '/lambda/foo1') self.assertEqual(request_context['accountId'], TEST_AWS_ACCOUNT_ID) self.assertEqual(request_context['resourceId'], resource.get('id')) self.assertEqual(request_context['stage'], self.TEST_STAGE_NAME) self.assertEqual(request_context['identity']['userAgent'], 'python-requests/testing') result = requests.delete(url, data=json.dumps(data)) self.assertEqual(result.status_code, 404)
def test_kinesis_lambda_sns_ddb_sqs_streams(self): def create_kinesis_stream(name, delete=False): stream = aws_stack.create_kinesis_stream(name, delete=delete) stream.wait_for() ddb_lease_table_suffix = "-kclapp" table_name = TEST_TABLE_NAME + "klsdss" + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME lambda_stream_name = "lambda-stream-%s" % short_uid() lambda_queue_name = "lambda-queue-%s" % short_uid() lambda_ddb_name = "lambda-ddb-%s" % short_uid() queue_name = "queue-%s" % short_uid() dynamodb = aws_stack.connect_to_resource("dynamodb") dynamodb_service = aws_stack.connect_to_service("dynamodb") dynamodbstreams = aws_stack.connect_to_service("dynamodbstreams") kinesis = aws_stack.connect_to_service("kinesis") sns = aws_stack.connect_to_service("sns") sqs = aws_stack.connect_to_service("sqs") LOGGER.info("Creating test streams...") run_safe( lambda: dynamodb_service.delete_table(TableName=stream_name + ddb_lease_table_suffix), print_error=False, ) create_kinesis_stream(stream_name, delete=True) create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix, ) LOGGER.info("Kinesis consumer initialized.") # create table with stream forwarding config aws_stack.create_dynamodb_table( table_name, partition_key=PARTITION_KEY, stream_view_type="NEW_AND_OLD_IMAGES", ) # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams["Streams"]: if stream["TableName"] == table_name: ddb_event_source_arn = stream["StreamArn"] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS) testutil.create_lambda_function( func_name=lambda_ddb_name, zip_file=zip_file, event_source_arn=ddb_event_source_arn, delete=True, ) # make sure we cannot create Lambda with same name twice with self.assertRaises(Exception): testutil.create_lambda_function( func_name=lambda_ddb_name, zip_file=zip_file, event_source_arn=ddb_event_source_arn, ) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )["StreamDescription"]["StreamARN"] testutil.create_lambda_function( func_name=lambda_stream_name, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, ) # deploy test lambda connected to SQS queue sqs_queue_info = testutil.create_sqs_queue(queue_name) testutil.create_lambda_function( func_name=lambda_queue_name, zip_file=zip_file, event_source_arn=sqs_queue_info["QueueArn"], ) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = (num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items) LOGGER.info("Putting %s items to table..." % num_events_ddb) table = dynamodb.Table(table_name) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: "testId%s" % i, "data": "foobar123" }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: "testId%s" % i, "data": "foobar123_put_existing" }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item( RequestItems={ table_name: [ { "PutRequest": { "Item": { PARTITION_KEY: short_uid(), "data": "foobar123 ✓" } } }, { "PutRequest": { "Item": { PARTITION_KEY: short_uid(), "data": "foobar123 £" } } }, { "PutRequest": { "Item": { PARTITION_KEY: short_uid(), "data": "foobar123 ¢" } } }, ] }) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item( TableName=table_name, Key={PARTITION_KEY: { "S": "testId%s" % i }}, AttributeUpdates={ "data": { "Action": "PUT", "Value": { "S": "foobar123_updated" } } }, ) # put items to stream num_events_kinesis = 1 num_kinesis_records = 10 LOGGER.info("Putting %s records in %s event to stream..." % (num_kinesis_records, num_events_kinesis)) kinesis.put_records( Records=[{ "Data": "{}", "PartitionKey": "testId%s" % i } for i in range(0, num_kinesis_records)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME, ) # put 1 item to stream that will trigger an error in the Lambda num_events_kinesis_err = 1 for i in range(num_events_kinesis_err): kinesis.put_record( Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey="testIdError", StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME, ) # create SNS topic, connect it to the Lambda, publish test messages num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe( TopicArn=response["TopicArn"], Protocol="lambda", Endpoint=aws_stack.lambda_function_arn(lambda_stream_name), ) for i in range(num_events_sns): sns.publish( TopicArn=response["TopicArn"], Subject="test_subject", Message="test message %s" % i, ) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id="shardId-000000000000", count=10) self.assertEqual(10, len(latest)) # send messages to SQS queue num_events_sqs = 4 for i in range(num_events_sqs): sqs.send_message(QueueUrl=sqs_queue_info["QueueUrl"], MessageBody=str(i)) LOGGER.info("Waiting some time before finishing test.") time.sleep(2) num_events_lambda = num_events_ddb + num_events_sns + num_events_sqs num_events = num_events_lambda + num_kinesis_records def check_events(): if len(events) != num_events: msg = "DynamoDB and Kinesis updates retrieved (actual/expected): %s/%s" % ( len(events), num_events, ) LOGGER.warning(msg) self.assertEqual(num_events, len(events)) event_items = [ json.loads(base64.b64decode(e["data"])) for e in events ] # make sure the we have the right amount of INSERT/MODIFY event types inserts = [ e for e in event_items if e.get("__action_type") == "INSERT" ] modifies = [ e for e in event_items if e.get("__action_type") == "MODIFY" ] self.assertEqual(num_put_new_items + num_batch_items, len(inserts)) self.assertEqual(num_put_existing_items + num_updates_ddb, len(modifies)) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=15, sleep=2) # check cloudwatch notifications def check_cw_invocations(): num_invocations = get_lambda_invocations_count(lambda_stream_name) expected_invocation_count = num_events_kinesis + num_events_kinesis_err + num_events_sns self.assertEqual(expected_invocation_count, num_invocations) num_error_invocations = get_lambda_invocations_count( lambda_stream_name, "Errors") self.assertEqual(num_events_kinesis_err, num_error_invocations) # Lambda invocations are running asynchronously, hence sleep some time here to wait for results retry(check_cw_invocations, retries=7, sleep=2) # clean up testutil.delete_lambda_function(lambda_stream_name) testutil.delete_lambda_function(lambda_ddb_name) testutil.delete_lambda_function(lambda_queue_name) sqs.delete_queue(QueueUrl=sqs_queue_info["QueueUrl"])
def test_lambda_streams_batch_and_transactions(self): ddb_lease_table_suffix = '-kclapp2' table_name = TEST_TABLE_NAME + 'lsbat' + ddb_lease_table_suffix stream_name = TEST_STREAM_NAME dynamodb = aws_stack.connect_to_service('dynamodb', client=True) dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=stream_name + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(stream_name, delete=True) events = [] # subscribe to inbound Kinesis stream def process_records(records, shard_id): events.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(stream_name, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config aws_stack.create_dynamodb_table(table_name, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == table_name: ddb_event_source_arn = stream['StreamArn'] self.assertTrue(ddb_event_source_arn) # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27, delete=True) # submit a batch with writes dynamodb.batch_write_item(RequestItems={table_name: [ {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId0'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId1'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId2'}, 'data': {'S': 'foobar123'}}}} ]}) # submit a batch with writes and deletes dynamodb.batch_write_item(RequestItems={table_name: [ {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId3'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId4'}, 'data': {'S': 'foobar123'}}}}, {'PutRequest': {'Item': {PARTITION_KEY: {'S': 'testId5'}, 'data': {'S': 'foobar123'}}}}, {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId0'}}}}, {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId1'}}}}, {'DeleteRequest': {'Key': {PARTITION_KEY: {'S': 'testId2'}}}}, ]}) # submit a transaction with writes and delete dynamodb.transact_write_items(TransactItems=[ {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123'}}}}, {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123'}}}}, {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId8'}, 'data': {'S': 'foobar123'}}}}, {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId3'}}}}, {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId4'}}}}, {'Delete': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId5'}}}}, ]) # submit a batch with a put over existing item dynamodb.transact_write_items(TransactItems=[ {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId6'}, 'data': {'S': 'foobar123_updated1'}}}}, ]) # submit a transaction with a put over existing item dynamodb.transact_write_items(TransactItems=[ {'Put': {'TableName': table_name, 'Item': {PARTITION_KEY: {'S': 'testId7'}, 'data': {'S': 'foobar123_updated1'}}}}, ]) # submit a transaction with updates dynamodb.transact_write_items(TransactItems=[ {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId6'}}, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': {'#0': 'data'}, 'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}}, {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId7'}}, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': {'#0': 'data'}, 'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}}, {'Update': {'TableName': table_name, 'Key': {PARTITION_KEY: {'S': 'testId8'}}, 'UpdateExpression': 'SET #0 = :0', 'ExpressionAttributeNames': {'#0': 'data'}, 'ExpressionAttributeValues': {':0': {'S': 'foobar123_updated2'}}}}, ]) LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_insert = 9 num_modify = 5 num_delete = 6 num_events = num_insert + num_modify + num_delete def check_events(): if len(events) != num_events: LOGGER.warning(('DynamoDB updates retrieved (actual/expected): %s/%s') % (len(events), num_events)) self.assertEqual(len(events), num_events) event_items = [json.loads(base64.b64decode(e['data'])) for e in events] # make sure the we have the right amount of expected event types inserts = [e for e in event_items if e.get('__action_type') == 'INSERT'] modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY'] removes = [e for e in event_items if e.get('__action_type') == 'REMOVE'] self.assertEqual(len(inserts), num_insert) self.assertEqual(len(modifies), num_modify) self.assertEqual(len(removes), num_delete) for i, event in enumerate(inserts): self.assertNotIn('old_image', event) self.assertEqual(inserts[i]['new_image'], {'id': 'testId%d' % i, 'data': 'foobar123'}) self.assertEqual(modifies[0]['old_image'], {'id': 'testId6', 'data': 'foobar123'}) self.assertEqual(modifies[0]['new_image'], {'id': 'testId6', 'data': 'foobar123_updated1'}) self.assertEqual(modifies[1]['old_image'], {'id': 'testId7', 'data': 'foobar123'}) self.assertEqual(modifies[1]['new_image'], {'id': 'testId7', 'data': 'foobar123_updated1'}) self.assertEqual(modifies[2]['old_image'], {'id': 'testId6', 'data': 'foobar123_updated1'}) self.assertEqual(modifies[2]['new_image'], {'id': 'testId6', 'data': 'foobar123_updated2'}) self.assertEqual(modifies[3]['old_image'], {'id': 'testId7', 'data': 'foobar123_updated1'}) self.assertEqual(modifies[3]['new_image'], {'id': 'testId7', 'data': 'foobar123_updated2'}) self.assertEqual(modifies[4]['old_image'], {'id': 'testId8', 'data': 'foobar123'}) self.assertEqual(modifies[4]['new_image'], {'id': 'testId8', 'data': 'foobar123_updated2'}) for i, event in enumerate(removes): self.assertEqual(event['old_image'], {'id': 'testId%d' % i, 'data': 'foobar123'}) self.assertNotIn('new_image', event) # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=9, sleep=3) # clean up testutil.delete_lambda_function(TEST_LAMBDA_NAME_DDB)
def install_elasticsearch(version=None): from localstack.services.es import versions if not version: version = ELASTICSEARCH_DEFAULT_VERSION version = get_elasticsearch_install_version(version) install_dir = get_elasticsearch_install_dir(version) installed_executable = os.path.join(install_dir, "bin", "elasticsearch") if not os.path.exists(installed_executable): log_install_msg("Elasticsearch (%s)" % version) es_url = versions.get_download_url(version) install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join(config.TMP_FOLDER, "localstack.%s" % os.path.basename(es_url)) download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob( os.path.join(install_dir_parent, "elasticsearch*")) if not elasticsearch_dir: raise Exception("Unable to find Elasticsearch folder in %s" % install_dir_parent) shutil.move(elasticsearch_dir[0], install_dir) for dir_name in ("data", "logs", "modules", "plugins", "config/scripts"): dir_path = os.path.join(install_dir, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # install default plugins for plugin in ELASTICSEARCH_PLUGIN_LIST: if is_alpine(): # https://github.com/pires/docker-elasticsearch/issues/56 os.environ["ES_TMPDIR"] = "/tmp" plugin_binary = os.path.join(install_dir, "bin", "elasticsearch-plugin") plugin_dir = os.path.join(install_dir, "plugins", plugin) if not os.path.exists(plugin_dir): LOG.info("Installing Elasticsearch plugin %s" % plugin) def try_install(): safe_run([plugin_binary, "install", "-b", plugin]) # We're occasionally seeing javax.net.ssl.SSLHandshakeException -> add download retries download_attempts = 3 try: retry(try_install, retries=download_attempts - 1, sleep=2) except Exception: LOG.warning( "Unable to download Elasticsearch plugin '%s' after %s attempts", plugin, download_attempts, ) if not os.environ.get("IGNORE_ES_DOWNLOAD_ERRORS"): raise # delete some plugins to free up space for plugin in ELASTICSEARCH_DELETE_MODULES: module_dir = os.path.join(install_dir, "modules", plugin) rm_rf(module_dir) # disable x-pack-ml plugin (not working on Alpine) xpack_dir = os.path.join(install_dir, "modules", "x-pack-ml", "platform") rm_rf(xpack_dir) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(install_dir, "config", "jvm.options") if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9\.]+$)", r"# \1", jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def test_lambda_subscribe_sns_topic(self): function_name = '{}-{}'.format(TEST_LAMBDA_FUNCTION_PREFIX, short_uid()) zip_file = testutil.create_lambda_archive( script=load_file(TEST_LAMBDA_ECHO_FILE), get_content=True, runtime=LAMBDA_RUNTIME_PYTHON36 ) testutil.create_lambda_function( zip_file=zip_file, func_name=function_name, runtime=LAMBDA_RUNTIME_PYTHON36 ) topic = self.sns_client.create_topic( Name=TEST_SNS_TOPIC_NAME ) topic_arn = topic['TopicArn'] self.sns_client.subscribe( TopicArn=topic_arn, Protocol='lambda', Endpoint=lambda_api.func_arn(function_name), ) subject = '[Subject] Test subject' message = 'Hello world.' self.sns_client.publish( TopicArn=topic_arn, Subject=subject, Message=message ) logs = aws_stack.connect_to_service('logs') def get_event_message(events): for event in events: raw_message = event['message'] if 'START' in raw_message or 'END' in raw_message or 'REPORT' in raw_message: continue return json.loads(raw_message) return None # wait for lambda executing def check_log_streams(): rs = logs.describe_log_streams( logGroupName='/aws/lambda/{}'.format(function_name) ) self.assertEqual(len(rs['logStreams']), 1) return rs['logStreams'][0]['logStreamName'] log_stream = retry(check_log_streams, retries=3, sleep=2) rs = logs.get_log_events( logGroupName='/aws/lambda/{}'.format(function_name), logStreamName=log_stream ) message = get_event_message(rs['events']) self.assertEqual(len(message['Records']), 1) notification = message['Records'][0]['Sns'] self.assertIn('Subject', notification) self.assertEqual(notification['Subject'], subject)
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table( TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis(TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME)['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # set number of items to update/put to table num_events_ddb = 15 num_put_new_items = 5 num_put_existing_items = 2 num_batch_items = 3 num_updates_ddb = num_events_ddb - num_put_new_items - num_put_existing_items - num_batch_items LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_put_new_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # Put items with an already existing ID (fix https://github.com/localstack/localstack/issues/522) for i in range(0, num_put_existing_items): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123_put_existing' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item(RequestItems={TEST_TABLE_NAME: [ {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 £'}}}, {'PutRequest': {'Item': {PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢'}}} ]}) # update some items, which also triggers notification events for i in range(0, num_updates_ddb): dynamodb_service.update_item(TableName=TEST_TABLE_NAME, Key={PARTITION_KEY: {'S': 'testId%s' % i}}, AttributeUpdates={'data': { 'Action': 'PUT', 'Value': {'S': 'foobar123_updated'} }}) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records( Records=[ { 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis) ], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME ) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe(TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records(TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns def check_events(): if len(EVENTS) != num_events: LOGGER.warning(('DynamoDB and Kinesis updates retrieved ' + '(actual/expected): %s/%s') % (len(EVENTS), num_events)) assert len(EVENTS) == num_events event_items = [json.loads(base64.b64decode(e['data'])) for e in EVENTS] inserts = [e for e in event_items if e.get('__action_type') == 'INSERT'] modifies = [e for e in event_items if e.get('__action_type') == 'MODIFY'] assert len(inserts) == num_put_new_items + num_batch_items assert len(modifies) == num_put_existing_items + num_updates_ddb # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=7, sleep=3) # make sure the we have the right amount of INSERT/MODIFY event types # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == num_events_ddb
def create_lambda_function(func_name, zip_file=None, event_source_arn=None, handler_file=None, handler=None, starting_position=None, runtime=None, envvars={}, tags={}, libs=[], delete=False, layers=None, **kwargs): """Utility method to create a new function via the Lambda API""" starting_position = starting_position or LAMBDA_DEFAULT_STARTING_POSITION runtime = runtime or LAMBDA_DEFAULT_RUNTIME client = aws_stack.connect_to_service('lambda') # load zip file content if handler_file is specified if not zip_file and handler_file: file_content = load_file(handler_file) if os.path.exists( handler_file) else handler_file if libs or not handler: zip_file = create_lambda_archive(file_content, libs=libs, get_content=True, runtime=runtime or LAMBDA_DEFAULT_RUNTIME) else: zip_file = create_zip_file(handler_file, get_content=True) handler = handler or LAMBDA_DEFAULT_HANDLER if delete: try: # Delete function if one already exists client.delete_function(FunctionName=func_name) except Exception: pass # create function additional_kwargs = kwargs kwargs = { 'FunctionName': func_name, 'Runtime': runtime, 'Handler': handler, 'Role': LAMBDA_TEST_ROLE, 'Code': { 'ZipFile': zip_file }, 'Timeout': LAMBDA_TIMEOUT_SEC, 'Environment': dict(Variables=envvars), 'Tags': tags } kwargs.update(additional_kwargs) if layers: kwargs['Layers'] = layers create_func_resp = client.create_function(**kwargs) resp = { 'CreateFunctionResponse': create_func_resp, 'CreateEventSourceMappingResponse': None } # create event source mapping if event_source_arn: resp[ 'CreateEventSourceMappingResponse'] = client.create_event_source_mapping( FunctionName=func_name, EventSourceArn=event_source_arn, StartingPosition=starting_position) return resp
def test_kinesis_lambda_sns_ddb_streams(): ddb_lease_table_suffix = '-kclapp' dynamodb = aws_stack.connect_to_resource('dynamodb') dynamodb_service = aws_stack.connect_to_service('dynamodb') dynamodbstreams = aws_stack.connect_to_service('dynamodbstreams') kinesis = aws_stack.connect_to_service('kinesis') sns = aws_stack.connect_to_service('sns') LOGGER.info('Creating test streams...') run_safe(lambda: dynamodb_service.delete_table(TableName=TEST_STREAM_NAME + ddb_lease_table_suffix), print_error=False) aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) aws_stack.create_kinesis_stream(TEST_LAMBDA_SOURCE_STREAM_NAME) # subscribe to inbound Kinesis stream def process_records(records, shard_id): EVENTS.extend(records) # start the KCL client process in the background kinesis_connector.listen_to_kinesis( TEST_STREAM_NAME, listener_func=process_records, wait_until_started=True, ddb_lease_table_suffix=ddb_lease_table_suffix) LOGGER.info('Kinesis consumer initialized.') # create table with stream forwarding config testutil.create_dynamodb_table(TEST_TABLE_NAME, partition_key=PARTITION_KEY, stream_view_type='NEW_AND_OLD_IMAGES') # list DDB streams and make sure the table stream is there streams = dynamodbstreams.list_streams() ddb_event_source_arn = None for stream in streams['Streams']: if stream['TableName'] == TEST_TABLE_NAME: ddb_event_source_arn = stream['StreamArn'] assert ddb_event_source_arn # deploy test lambda connected to DynamoDB Stream zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # make sure we cannot create Lambda with same name twice assert_raises(Exception, testutil.create_lambda_function, func_name=TEST_LAMBDA_NAME_DDB, zip_file=zip_file, event_source_arn=ddb_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # deploy test lambda connected to Kinesis Stream kinesis_event_source_arn = kinesis.describe_stream( StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME )['StreamDescription']['StreamARN'] testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_STREAM, zip_file=zip_file, event_source_arn=kinesis_event_source_arn, runtime=LAMBDA_RUNTIME_PYTHON27) # put items to table num_events_ddb = 10 LOGGER.info('Putting %s items to table...' % num_events_ddb) table = dynamodb.Table(TEST_TABLE_NAME) for i in range(0, num_events_ddb - 3): table.put_item(Item={ PARTITION_KEY: 'testId%s' % i, 'data': 'foobar123' }) # batch write some items containing non-ASCII characters dynamodb.batch_write_item( RequestItems={ TEST_TABLE_NAME: [{ 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ✓' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 £' } } }, { 'PutRequest': { 'Item': { PARTITION_KEY: short_uid(), 'data': 'foobar123 ¢' } } }] }) # put items to stream num_events_kinesis = 10 LOGGER.info('Putting %s items to stream...' % num_events_kinesis) kinesis.put_records(Records=[{ 'Data': '{}', 'PartitionKey': 'testId%s' % i } for i in range(0, num_events_kinesis)], StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # put 1 item to stream that will trigger an error in the Lambda kinesis.put_record(Data='{"%s": 1}' % lambda_integration.MSG_BODY_RAISE_ERROR_FLAG, PartitionKey='testIderror', StreamName=TEST_LAMBDA_SOURCE_STREAM_NAME) # create SNS topic, connect it to the Lambda, publish test message num_events_sns = 3 response = sns.create_topic(Name=TEST_TOPIC_NAME) sns.subscribe( TopicArn=response['TopicArn'], Protocol='lambda', Endpoint=aws_stack.lambda_function_arn(TEST_LAMBDA_NAME_STREAM)) for i in range(0, num_events_sns): sns.publish(TopicArn=response['TopicArn'], Message='test message %s' % i) # get latest records latest = aws_stack.kinesis_get_latest_records( TEST_LAMBDA_SOURCE_STREAM_NAME, shard_id='shardId-000000000000', count=10) assert len(latest) == 10 LOGGER.info('Waiting some time before finishing test.') time.sleep(2) num_events = num_events_ddb + num_events_kinesis + num_events_sns def check_events(): if len(EVENTS) != num_events: LOGGER.warning( ('DynamoDB and Kinesis updates retrieved ' + '(actual/expected): %s/%s') % (len(EVENTS), num_events)) assert len(EVENTS) == num_events # this can take a long time in CI, make sure we give it enough time/retries retry(check_events, retries=7, sleep=3) # check cloudwatch notifications stats1 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM) assert len(stats1['Datapoints']) == 2 + num_events_sns stats2 = get_lambda_metrics(TEST_LAMBDA_NAME_STREAM, 'Errors') assert len(stats2['Datapoints']) == 1 stats3 = get_lambda_metrics(TEST_LAMBDA_NAME_DDB) assert len(stats3['Datapoints']) == 10
def test_prime_and_destroy_containers(): # run these tests only for the "reuse containers" Lambda executor if not isinstance(lambda_api.LAMBDA_EXECUTOR, lambda_executors.LambdaExecutorReuseContainers): return executor = lambda_api.LAMBDA_EXECUTOR func_name = 'test_prime_and_destroy_containers' # create a new lambda lambda_client = aws_stack.connect_to_service('lambda') func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone executor.cleanup() assert len(executor.get_all_container_names()) == 0 # deploy and invoke lambda without Docker zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_ENV), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=func_name, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27, envvars={'Hello': 'World'}) assert len(executor.get_all_container_names()) == 0 assert executor.function_invoke_times == {} # invoke a few times. durations = [] num_iterations = 3 for i in range(0, num_iterations + 1): prev_invoke_time = None if i > 0: prev_invoke_time = executor.function_invoke_times[func_arn] start_time = time.time() lambda_client.invoke(FunctionName=func_name, Payload=b'{}') duration = time.time() - start_time assert len(executor.get_all_container_names()) == 1 # ensure the last invoke time is being updated properly. if i > 0: assert executor.function_invoke_times[func_arn] > prev_invoke_time else: assert executor.function_invoke_times[func_arn] > 0 durations.append(duration) # the first call would have created the container. subsequent calls would reuse and be faster. for i in range(1, num_iterations + 1): assert durations[i] < durations[0] status = executor.get_docker_container_status(func_arn) assert status == 1 executor.cleanup() status = executor.get_docker_container_status(func_arn) assert status == 0 assert len(executor.get_all_container_names()) == 0
def create_function(cls, file, name, runtime=None, libs=None): runtime = runtime or LAMBDA_RUNTIME_PYTHON27 zip_file = testutil.create_lambda_archive( load_file(file), get_content=True, libs=libs, runtime=runtime) testutil.create_lambda_function( func_name=name, zip_file=zip_file, runtime=runtime)
def test_lambda_runtimes(): lambda_client = aws_stack.connect_to_service('lambda') # deploy and invoke lambda - Python 2.7 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27) # Invocation Type not set result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # Invocation Type - RequestResponse result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}', InvocationType='RequestResponse') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # Invocation Type - Event result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}', InvocationType='Event') assert result['StatusCode'] == 202 # Invocation Type - DryRun result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}', InvocationType='DryRun') assert result['StatusCode'] == 204 if use_docker(): # deploy and invoke lambda - Python 3.6 zip_file = testutil.create_lambda_archive( load_file(TEST_LAMBDA_PYTHON3), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java if not os.path.exists(TEST_LAMBDA_JAVA): mkdir(os.path.dirname(TEST_LAMBDA_JAVA)) download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA) # Lambda supports single JAR deployments without the zip, so we upload the JAR directly. test_java_jar = load_file(TEST_LAMBDA_JAVA, mode='rb') assert test_java_jar is not None testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA, zip_file=test_java_jar, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'LinkedHashMap' in to_str(result_data) # test SNSEvent result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}') assert result['StatusCode'] == 202 # test DDBEvent result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"dynamodb": {"Message": "{}"}}]}') assert result['StatusCode'] == 202 # test KinesisEvent result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JAVA, Payload= b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}' ) assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'KinesisEvent' in to_str(result_data) # deploy and invoke lambda - Java with stream handler testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=test_java_jar, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java with serializable input object testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, zip_file=test_java_jar, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.SerializedInputLambdaHandler') result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, Payload=b'{"bucket": "test_bucket", "key": "test_key"}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == { 'validated': True, 'bucket': 'test_bucket', 'key': 'test_key' } if use_docker(): # deploy and invoke lambda - Node.js zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke - .NET Core 2.0. Its already a zip zip_file = TEST_LAMBDA_DOTNETCORE2 zip_file_content = None with open(zip_file, 'rb') as file_obj: zip_file_content = file_obj.read() testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_DOTNETCORE2, zip_file=zip_file_content, handler= 'DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler', runtime=LAMBDA_RUNTIME_DOTNETCORE2) result = lambda_client.invoke( FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}'
def test_create_delete_stack(self): cloudformation = aws_stack.connect_to_resource('cloudformation') cf_client = aws_stack.connect_to_service('cloudformation') s3 = aws_stack.connect_to_service('s3') sns = aws_stack.connect_to_service('sns') sqs = aws_stack.connect_to_service('sqs') apigateway = aws_stack.connect_to_service('apigateway') template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1)) # deploy template stack_name = 'stack-%s' % short_uid() cloudformation.create_stack(StackName=stack_name, TemplateBody=template) _await_stack_completion(stack_name) # assert that resources have been created assert bucket_exists('cf-test-bucket-1') queue_url = queue_exists('cf-test-queue-1') assert queue_url topic_arn = topic_exists('%s-test-topic-1-1' % stack_name) assert topic_arn assert stream_exists('cf-test-stream-1') resource = describe_stack_resource(stack_name, 'SQSQueueNoNameProperty') assert queue_exists(resource['PhysicalResourceId']) assert ssm_param_exists('cf-test-param-1') # assert that tags have been created tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet'] self.assertEqual(tags, [{'Key': 'foobar', 'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')}]) tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags'] self.assertEqual(tags, [ {'Key': 'foo', 'Value': 'cf-test-bucket-1'}, {'Key': 'bar', 'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')} ]) queue_tags = sqs.list_queue_tags(QueueUrl=queue_url) self.assertIn('Tags', queue_tags) self.assertEqual(queue_tags['Tags'], {'key1': 'value1', 'key2': 'value2'}) # assert that bucket notifications have been created notifications = s3.get_bucket_notification_configuration(Bucket='cf-test-bucket-1') self.assertIn('QueueConfigurations', notifications) self.assertIn('LambdaFunctionConfigurations', notifications) self.assertEqual(notifications['QueueConfigurations'][0]['QueueArn'], 'aws:arn:sqs:test:testqueue') self.assertEqual(notifications['QueueConfigurations'][0]['Events'], ['s3:ObjectDeleted:*']) self.assertEqual( notifications['LambdaFunctionConfigurations'][0]['LambdaFunctionArn'], 'aws:arn:lambda:test:testfunc' ) self.assertEqual(notifications['LambdaFunctionConfigurations'][0]['Events'], ['s3:ObjectCreated:*']) # assert that subscriptions have been created subs = sns.list_subscriptions()['Subscriptions'] subs = [s for s in subs if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']] self.assertEqual(len(subs), 1) self.assertIn(':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name), subs[0]['TopicArn']) # assert that subscription attributes are added properly attrs = sns.get_subscription_attributes(SubscriptionArn=subs[0]['SubscriptionArn'])['Attributes'] self.assertEqual(attrs, {'Endpoint': subs[0]['Endpoint'], 'Protocol': 'sqs', 'SubscriptionArn': subs[0]['SubscriptionArn'], 'TopicArn': subs[0]['TopicArn'], 'FilterPolicy': json.dumps({'eventType': ['created']})}) # assert that Gateway responses have been created test_api_name = 'test-api' api = [a for a in apigateway.get_rest_apis()['items'] if a['name'] == test_api_name][0] responses = apigateway.get_gateway_responses(restApiId=api['id'])['items'] self.assertEqual(len(responses), 2) types = [r['responseType'] for r in responses] self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX'])) # delete the stack cf_client.delete_stack(StackName=stack_name) # assert that resources have been deleted assert not bucket_exists('cf-test-bucket-1') assert not queue_exists('cf-test-queue-1') assert not topic_exists('%s-test-topic-1-1' % stack_name) retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def load_template_raw(tmpl_path: str) -> str: template = load_file( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "templates", tmpl_path) ) return template
def test_lambda_runtimes(): lambda_client = aws_stack.connect_to_service('lambda') # deploy and invoke lambda - Python 2.7 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' if use_docker(): # deploy and invoke lambda - Python 3.6 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON3), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java if not os.path.exists(TEST_LAMBDA_JAVA): mkdir(os.path.dirname(TEST_LAMBDA_JAVA)) download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA) zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'LinkedHashMap' in to_str(result_data) # test SNSEvent result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'async': 'True'} # test DDBEvent result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"dynamodb": {"Message": "{}"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'async': 'True'} # test KinesisEvent result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'KinesisEvent' in to_str(result_data) # deploy and invoke lambda - Java with stream handler testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java with serializable input object testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.SerializedInputLambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, Payload=b'{"bucket": "test_bucket", "key": "test_key"}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'validated': True, 'bucket': 'test_bucket', 'key': 'test_key'} if use_docker(): # deploy and invoke lambda - Node.js zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke - .NET Core 2.0. Its already a zip zip_file = TEST_LAMBDA_DOTNETCORE2 zip_file_content = None with open(zip_file, 'rb') as file_obj: zip_file_content = file_obj.read() testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DOTNETCORE2, zip_file=zip_file_content, handler='DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler', runtime=LAMBDA_RUNTIME_DOTNETCORE2) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}'
def test_validate_template(self): cloudformation = aws_stack.connect_to_service('cloudformation') template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1)) response = cloudformation.validate_template(TemplateBody=template) self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'], 200)