def install_dynamodb_local():
if not os.path.exists(INSTALL_DIR_DDB):
LOGGER.info('Downloading and installing local DynamoDB server. This may take some time.')
mkdir(INSTALL_DIR_DDB)
# download and extract archive
download_and_extract_with_retry(DYNAMODB_JAR_URL, TMP_ARCHIVE_DDB, INSTALL_DIR_DDB)
# fix for Alpine, otherwise DynamoDBLocal fails with:
# DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found
if is_alpine():
ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB
patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir
if not os.path.exists(patched_marker):
patched_lib = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +
'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so')
patched_jar = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' +
'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar')
run("curl -L -o %s/libsqlite4java-linux-amd64.so '%s'" % (ddb_libs_dir, patched_lib))
run("curl -L -o %s/sqlite4java.jar '%s'" % (ddb_libs_dir, patched_jar))
save_file(patched_marker, '')
# fix logging configuration for DynamoDBLocal
log4j2_config = """<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>"""
log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml')
save_file(log4j2_file, log4j2_config)
run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)
def setUpClass(cls):
cls.lambda_client = aws_stack.connect_to_service('lambda')
# deploy lambda - Java
if not os.path.exists(TEST_LAMBDA_JAVA):
mkdir(os.path.dirname(TEST_LAMBDA_JAVA))
download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA)
# Lambda supports single JAR deployments without the zip,
# so we upload the JAR directly.
cls.test_java_jar = load_file(TEST_LAMBDA_JAVA, mode='rb')
cls.test_java_zip = testutil.create_zip_file(TEST_LAMBDA_JAVA,
get_content=True)
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JAVA,
zip_file=cls.test_java_jar,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.LambdaHandler')
# deploy lambda - Java with stream handler
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JAVA_STREAM,
zip_file=cls.test_java_jar,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.LambdaStreamHandler')
# deploy lambda - Java with serializable input object
testutil.create_lambda_function(
func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE,
zip_file=cls.test_java_zip,
runtime=LAMBDA_RUNTIME_JAVA8,
handler='cloud.localstack.sample.SerializedInputLambdaHandler')
def start_elasticsearch(port=None, delete_data=True, asynchronous=False, update_listener=None):
port = port or config.PORT_ELASTICSEARCH
# delete Elasticsearch data that may be cached locally from a previous test run
delete_all_elasticsearch_data()
install.install_elasticsearch()
backend_port = DEFAULT_PORT_ELASTICSEARCH_BACKEND
es_data_dir = '%s/infra/elasticsearch/data' % (ROOT_PATH)
es_tmp_dir = '%s/infra/elasticsearch/tmp' % (ROOT_PATH)
if config.DATA_DIR:
es_data_dir = '%s/elasticsearch' % config.DATA_DIR
# Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments,
# hence we use the default bind address 127.0.0.0 and put a proxy in front of it
cmd = (('%s/infra/elasticsearch/bin/elasticsearch ' +
'-E http.port=%s -E http.publish_port=%s -E http.compression=false -E path.data=%s') %
(ROOT_PATH, backend_port, backend_port, es_data_dir))
env_vars = {
'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'),
'ES_TMPDIR': es_tmp_dir
}
print('Starting local Elasticsearch (%s port %s)...' % (get_service_protocol(), port))
if delete_data:
run('rm -rf %s' % es_data_dir)
# fix permissions
chmod_r('%s/infra/elasticsearch' % ROOT_PATH, 0o777)
mkdir(es_data_dir)
chmod_r(es_data_dir, 0o777)
# start proxy and ES process
start_proxy_for_service('elasticsearch', port, backend_port,
update_listener, quiet=True, params={'protocol_version': 'HTTP/1.0'})
if is_root():
cmd = "su -c '%s' localstack" % cmd
thread = do_run(cmd, asynchronous, env_vars=env_vars)
return thread
def get_lambda_code_param(params, **kwargs):
code = params.get("Code", {})
zip_file = code.get("ZipFile")
if zip_file and not is_base64(zip_file):
tmp_dir = new_tmp_dir()
handler_file = get_handler_file_from_name(
params["Handler"], runtime=params["Runtime"])
tmp_file = os.path.join(tmp_dir, handler_file)
save_file(tmp_file, zip_file)
# add 'cfn-response' module to archive - see:
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-lambda-function-code-cfnresponsemodule.html
cfn_response_tmp_file = get_cfn_response_mod_file()
cfn_response_mod_dir = os.path.join(tmp_dir, "node_modules",
"cfn-response")
mkdir(cfn_response_mod_dir)
cp_r(
cfn_response_tmp_file,
os.path.join(cfn_response_mod_dir, "index.js"),
)
# create zip file
zip_file = create_zip_file(tmp_dir, get_content=True)
code["ZipFile"] = zip_file
rm_rf(tmp_dir)
return code
def download_and_extract(archive_url,
target_dir,
retries=0,
sleep=3,
tmp_archive=None):
mkdir(target_dir)
tmp_archive = tmp_archive or new_tmp_file()
if not os.path.exists(tmp_archive):
# create temporary placeholder file, to avoid duplicate parallel downloads
save_file(tmp_archive, '')
for i in range(retries + 1):
try:
download(archive_url, tmp_archive)
break
except Exception:
time.sleep(sleep)
_, ext = os.path.splitext(tmp_archive)
if ext == '.zip':
unzip(tmp_archive, target_dir)
elif ext == '.gz' or ext == '.bz2':
untar(tmp_archive, target_dir)
else:
raise Exception('Unsupported archive format: %s' % ext)
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
log_install_msg('Step Functions')
mkdir(INSTALL_DIR_STEPFUNCTIONS)
run('{dc} pull {img}'.format(dc=config.DOCKER_CMD,
img=IMAGE_NAME_SFN_LOCAL))
docker_name = 'tmp-ls-sfn'
run(('{dc} run --name={dn} --entrypoint= -d --rm {img} sleep 15'
).format(dc=config.DOCKER_CMD,
dn=docker_name,
img=IMAGE_NAME_SFN_LOCAL))
time.sleep(5)
run('{dc} cp {dn}:/home/stepfunctionslocal/ {tgt}'.format(
dc=config.DOCKER_CMD, dn=docker_name, tgt=INSTALL_DIR_INFRA))
run('mv %s/stepfunctionslocal/*.jar %s' %
(INSTALL_DIR_INFRA, INSTALL_DIR_STEPFUNCTIONS))
rm_rf('%s/stepfunctionslocal' % INSTALL_DIR_INFRA)
# apply patches
patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, SFN_PATCH_CLASS)
if not os.path.exists(patch_class_file):
download(SFN_PATCH_CLASS_URL, patch_class_file)
cmd = 'cd "%s"; zip %s %s' % (INSTALL_DIR_STEPFUNCTIONS,
INSTALL_PATH_STEPFUNCTIONS_JAR,
SFN_PATCH_CLASS)
run(cmd)
def create_lambda_archive(
script: str,
get_content: bool = False,
libs: List[str] = None,
runtime: str = None,
file_name: str = None,
exclude_func: Callable[[str], bool] = None,
):
"""Utility method to create a Lambda function archive"""
if libs is None:
libs = []
runtime = runtime or LAMBDA_DEFAULT_RUNTIME
with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir:
file_name = file_name or get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER, runtime=runtime)
script_file = os.path.join(tmp_dir, file_name)
if os.path.sep in script_file:
mkdir(os.path.dirname(script_file))
# create __init__.py files along the path to allow Python imports
path = file_name.split(os.path.sep)
for i in range(1, len(path)):
save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "")
save_file(script_file, script)
chmod_r(script_file, 0o777)
# copy libs
for lib in libs:
paths = [lib, "%s.py" % lib]
try:
module = importlib.import_module(lib)
paths.append(module.__file__)
except Exception:
pass
target_dir = tmp_dir
root_folder = os.path.join(LOCALSTACK_VENV_FOLDER, "lib/python*/site-packages")
if lib == "localstack":
paths = ["localstack/*.py", "localstack/utils"]
root_folder = LOCALSTACK_ROOT_FOLDER
target_dir = os.path.join(tmp_dir, lib)
mkdir(target_dir)
for path in paths:
file_to_copy = path if path.startswith("/") else os.path.join(root_folder, path)
for file_path in glob.glob(file_to_copy):
name = os.path.join(target_dir, file_path.split(os.path.sep)[-1])
if os.path.isdir(file_path):
copy_dir(file_path, name)
else:
shutil.copyfile(file_path, name)
if exclude_func:
for dirpath, folders, files in os.walk(tmp_dir):
for name in list(folders) + list(files):
full_name = os.path.join(dirpath, name)
relative = os.path.relpath(full_name, start=tmp_dir)
if exclude_func(relative):
rm_rf(full_name)
# create zip file
result = create_zip_file(tmp_dir, get_content=get_content)
return result
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None):
kinesis_mock_bin = install.install_kinesis_mock()
backend_port = get_free_tcp_port()
global PORT_KINESIS_BACKEND
PORT_KINESIS_BACKEND = backend_port
kinesis_data_dir_param = ""
if config.DATA_DIR:
kinesis_data_dir = "%s/kinesis" % config.DATA_DIR
mkdir(kinesis_data_dir)
kinesis_data_dir_param = "SHOULD_PERSIST_DATA=true PERSIST_PATH=%s" % kinesis_data_dir
if not config.LS_LOG:
log_level = "INFO"
elif config.LS_LOG == "warning":
log_level = "WARN"
else:
log_level = config.LS_LOG.upper()
log_level_param = "LOG_LEVEL=%s" % log_level
latency = config.KINESIS_LATENCY + "ms"
latency_param = (
"CREATE_STREAM_DURATION={l} DELETE_STREAM_DURATION={l} REGISTER_STREAM_CONSUMER_DURATION={l} "
"START_STREAM_ENCRYPTION_DURATION={l} STOP_STREAM_ENCRYPTION_DURATION={l} "
"DEREGISTER_STREAM_CONSUMER_DURATION={l} MERGE_SHARDS_DURATION={l} SPLIT_SHARD_DURATION={l} "
"UPDATE_SHARD_COUNT_DURATION={l}").format(l=latency)
if config.KINESIS_INITIALIZE_STREAMS != "":
initialize_streams_param = "INITIALIZE_STREAMS=%s" % config.KINESIS_INITIALIZE_STREAMS
else:
initialize_streams_param = ""
if kinesis_mock_bin.endswith(".jar"):
cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s" % (
backend_port,
config.KINESIS_SHARD_LIMIT,
latency_param,
kinesis_data_dir_param,
log_level_param,
initialize_streams_param,
kinesis_mock_bin,
)
else:
chmod_r(kinesis_mock_bin, 0o777)
cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1" % (
backend_port,
config.KINESIS_SHARD_LIMIT,
latency_param,
kinesis_data_dir_param,
log_level_param,
initialize_streams_param,
kinesis_mock_bin,
)
return _run_proxy_and_command(
cmd=cmd,
port=port,
backend_port=backend_port,
update_listener=update_listener,
asynchronous=asynchronous,
)
def install_elasticmq():
if not os.path.exists(INSTALL_DIR_ELASTICMQ):
LOGGER.info('Downloading and installing local ElasticMQ server. This may take some time.')
mkdir(INSTALL_DIR_ELASTICMQ)
# download archive
if not os.path.exists(TMP_ARCHIVE_ELASTICMQ):
download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ)
shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)
def install_local_kms():
binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace('<arch>', get_arch())
if not os.path.exists(binary_path):
log_install_msg('KMS')
mkdir(INSTALL_DIR_KMS)
kms_url = KMS_URL_PATTERN.replace('<arch>', get_arch())
download(kms_url, binary_path)
chmod_r(binary_path, 0o777)
def test_get_java_lib_folder_classpath_no_directories(self):
base_dir = new_tmp_dir()
jar_file = os.path.join(base_dir, 'foo.jar')
save_file(jar_file, '')
lib_file = os.path.join(base_dir, 'lib', 'lib.jar')
mkdir(os.path.dirname(lib_file))
save_file(lib_file, '')
self.assertEquals('.:foo.jar:lib/lib.jar', lambda_executors.Util.get_java_classpath(jar_file))
def start_elasticsearch(port=None,
version=None,
delete_data=True,
asynchronous=False,
update_listener=None):
if STATE.get('_thread_'):
return STATE['_thread_']
port = port or config.PORT_ELASTICSEARCH
# delete Elasticsearch data that may be cached locally from a previous test run
delete_all_elasticsearch_data(version)
install.install_elasticsearch(version)
backend_port = get_free_tcp_port()
base_dir = install.get_elasticsearch_install_dir(version)
es_data_dir = os.path.join(base_dir, 'data')
es_tmp_dir = os.path.join(base_dir, 'tmp')
es_mods_dir = os.path.join(base_dir, 'modules')
if config.DATA_DIR:
delete_data = False
es_data_dir = '%s/elasticsearch' % config.DATA_DIR
# Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments,
# hence we use the default bind address 127.0.0.0 and put a proxy in front of it
backup_dir = os.path.join(config.TMP_FOLDER, 'es_backup')
cmd = (
('%s/bin/elasticsearch ' +
'-E http.port=%s -E http.publish_port=%s -E http.compression=false ' +
'-E path.data=%s -E path.repo=%s') %
(base_dir, backend_port, backend_port, es_data_dir, backup_dir))
if os.path.exists(os.path.join(es_mods_dir, 'x-pack-ml')):
cmd += ' -E xpack.ml.enabled=false'
env_vars = {
'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'),
'ES_TMPDIR': es_tmp_dir
}
LOG.debug('Starting local Elasticsearch (%s port %s)' %
(get_service_protocol(), port))
if delete_data:
rm_rf(es_data_dir)
# fix permissions
chmod_r(base_dir, 0o777)
mkdir(es_data_dir)
chmod_r(es_data_dir, 0o777)
mkdir(es_tmp_dir)
chmod_r(es_tmp_dir, 0o777)
# start proxy and ES process
proxy = start_proxy_for_service('elasticsearch',
port,
backend_port,
update_listener,
quiet=True,
params={'protocol_version': 'HTTP/1.0'})
STATE['_proxy_'] = proxy
if is_root():
cmd = "su localstack -c '%s'" % cmd
thread = do_run(cmd, asynchronous, env_vars=env_vars)
STATE['_thread_'] = thread
return thread
def _check(fname, is_dir):
test_entry = os.path.join(tmp_dir, fname)
mkdir(test_entry) if is_dir else save_file(test_entry,
"test content")
assert not is_empty_dir(tmp_dir)
assert is_empty_dir(tmp_dir,
ignore_hidden=True) == (fname == ".hidden")
rm_rf(test_entry)
assert is_empty_dir(tmp_dir)
def install_local_kms():
local_arch = f"{platform.system().lower()}-{get_arch()}"
binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", local_arch)
if not os.path.exists(binary_path):
log_install_msg("KMS")
mkdir(INSTALL_DIR_KMS)
kms_url = KMS_URL_PATTERN.replace("<arch>", local_arch)
download(kms_url, binary_path)
chmod_r(binary_path, 0o777)
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None):
target_dir = os.path.join(INSTALL_DIR_INFRA, 'kinesis-mock')
machine = platform.machine().lower()
system = platform.system().lower()
if machine == 'x86_64' or machine == 'amd64':
if system == 'windows':
target_file_name = 'kinesis-mock-mostly-static.exe'
elif system == 'linux':
target_file_name = 'kinesis-mock-linux-amd64-static'
elif system == 'darwin':
target_file_name = 'kinesis-mock-macos-amd64-dynamic'
else:
target_file_name = 'kinesis-mock.jar'
else:
target_file_name = 'kinesis-mock.jar'
target_file = os.path.join(target_dir, target_file_name)
if not os.path.exists(target_file):
response = requests.get(KINESIS_MOCK_RELEASES)
content = json.loads(to_str(response.content))
assets = content.get('assets', [])
filtered = [x for x in assets if x['name'] == target_file_name]
archive_url = filtered[0].get('browser_download_url')
download(archive_url, target_file)
port = port or config.PORT_KINESIS
backend_port = get_free_tcp_port()
kinesis_data_dir_param = ''
if config.DATA_DIR:
kinesis_data_dir = '%s/kinesis' % config.DATA_DIR
mkdir(kinesis_data_dir)
kinesis_data_dir_param = 'SHOULD_PERSIST_DATA=true PERSIST_PATH=%s' % kinesis_data_dir
if not config.LS_LOG:
log_level = 'INFO'
elif config.LS_LOG == 'warning':
log_level = 'WARN'
else:
log_level = config.LS_LOG.upper
log_level_param = 'LOG_LEVEL=%s' % (log_level)
latency = config.KINESIS_LATENCY + 'ms'
latency_param = 'CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s ' \
'START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s ' \
'DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s ' \
'UPDATE_SHARD_COUNT_DURATION=%s' \
% (latency, latency, latency, latency, latency, latency, latency, latency, latency)
if target_file_name.endswith('.jar'):
cmd = 'KINESIS_MOCK_HTTP1_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s java -XX:+UseG1GC -jar %s' \
% (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param,
log_level_param, target_file)
else:
chmod_r(target_file, 0o777)
cmd = 'KINESIS_MOCK_HTTP1_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s --gc=G1' \
% (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param,
log_level_param, target_file)
start_proxy_for_service('kinesis', port, backend_port, update_listener)
return do_run(cmd, asynchronous)
def install_elasticmq():
if not os.path.exists(INSTALL_DIR_ELASTICMQ):
log_install_msg('ElasticMQ')
mkdir(INSTALL_DIR_ELASTICMQ)
# download archive
tmp_archive = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar')
if not os.path.exists(tmp_archive):
download(ELASTICMQ_JAR_URL, tmp_archive)
shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def install_elasticmq():
# TODO remove this function if we stop using ElasticMQ entirely
if not os.path.exists(INSTALL_PATH_ELASTICMQ_JAR):
log_install_msg("ElasticMQ")
mkdir(INSTALL_DIR_ELASTICMQ)
# download archive
tmp_archive = os.path.join(config.dirs.tmp, "elasticmq-server.jar")
if not os.path.exists(tmp_archive):
download(ELASTICMQ_JAR_URL, tmp_archive)
shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def install_local_kms():
local_arch = get_os()
binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", local_arch)
if not os.path.exists(binary_path):
log_install_msg("KMS")
mkdir(INSTALL_DIR_KMS)
# TODO ARM download platform specific binary
kms_url = KMS_URL_PATTERN.replace("<arch>", local_arch)
download(kms_url, binary_path)
chmod_r(binary_path, 0o777)
def install_elasticmq():
if not os.path.exists(INSTALL_DIR_ELASTICMQ):
LOGGER.info(
'Downloading and installing local ElasticMQ server. This may take some time.'
)
mkdir(INSTALL_DIR_ELASTICMQ)
# download archive
if not os.path.exists(TMP_ARCHIVE_ELASTICMQ):
download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ)
shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)
def get_lambda_code(func_name,
retries=1,
cache_time=None,
env=None,
region=None):
if MOCK_OBJ:
return ""
env = aws_stack.get_environment(env)
if cache_time is None and not aws_stack.is_local_env(env):
cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT
lambda_client = _connect("lambda", env=env, region=region)
out = lambda_client.get_function(FunctionName=func_name)
loc = out["Code"]["Location"]
hash = md5(loc)
folder = TMP_DOWNLOAD_FILE_PATTERN.replace("*", hash)
filename = "archive.zip"
archive = "%s/%s" % (folder, filename)
try:
mkdir(folder)
if not os.path.isfile(archive):
download(loc, archive, verify_ssl=False)
if len(os.listdir(folder)) <= 1:
zip_path = os.path.join(folder, filename)
unzip(zip_path, folder)
except Exception as e:
print("WARN: %s" % e)
rm_rf(archive)
if retries > 0:
return get_lambda_code(func_name,
retries=retries - 1,
cache_time=1,
env=env)
else:
print("WARNING: Unable to retrieve lambda code: %s" % e)
# traverse subdirectories and get script sources
result = {}
for root, subdirs, files in os.walk(folder):
for file in files:
prefix = root.split(folder)[-1]
key = "%s/%s" % (prefix, file)
if re.match(r".+\.py$", key) or re.match(r".+\.js$", key):
codefile = "%s/%s" % (root, file)
result[key] = load_file(codefile)
# cleanup cache
clean_cache(
file_pattern=TMP_DOWNLOAD_FILE_PATTERN,
last_clean_time=last_cache_cleanup_time,
max_age=TMP_DOWNLOAD_CACHE_MAX_AGE,
)
# TODO: delete only if cache_time is over
rm_rf(folder)
return result
def test_get_java_lib_folder_classpath_no_directories(self):
base_dir = new_tmp_dir()
jar_file = os.path.join(base_dir, "foo.jar")
save_file(jar_file, "")
lib_file = os.path.join(base_dir, "lib", "lib.jar")
mkdir(os.path.dirname(lib_file))
save_file(lib_file, "")
classpath = lambda_executors.Util.get_java_classpath(jar_file)
self.assertIn(":foo.jar", classpath)
self.assertIn("lib/lib.jar:", classpath)
self.assertIn(":*.jar", classpath)
def install_kinesis_mock():
target_dir = INSTALL_PATH_KINESIS_MOCK
machine = platform.machine().lower()
system = platform.system().lower()
version = platform.version().lower()
is_probably_m1 = system == "darwin" and ("arm64" in version
or "arm32" in version)
LOG.debug("getting kinesis-mock for %s %s", system, machine)
if is_env_true("KINESIS_MOCK_FORCE_JAVA"):
# sometimes the static binaries may have problems, and we want to fal back to Java
bin_file = "kinesis-mock.jar"
elif (machine == "x86_64" or machine == "amd64") and not is_probably_m1:
if system == "windows":
bin_file = "kinesis-mock-mostly-static.exe"
elif system == "linux":
bin_file = "kinesis-mock-linux-amd64-static"
elif system == "darwin":
bin_file = "kinesis-mock-macos-amd64-dynamic"
else:
bin_file = "kinesis-mock.jar"
else:
bin_file = "kinesis-mock.jar"
bin_file_path = os.path.join(target_dir, bin_file)
if os.path.exists(bin_file_path):
LOG.debug("kinesis-mock found at %s", bin_file_path)
return bin_file_path
response = requests.get(KINESIS_MOCK_RELEASE_URL)
if not response.ok:
raise ValueError("Could not get list of releases from %s: %s" %
(KINESIS_MOCK_RELEASE_URL, response.text))
github_release = response.json()
download_url = None
for asset in github_release.get("assets", []):
# find the correct binary in the release
if asset["name"] == bin_file:
download_url = asset["browser_download_url"]
break
if download_url is None:
raise ValueError("could not find required binary %s in release %s" %
(bin_file, KINESIS_MOCK_RELEASE_URL))
mkdir(target_dir)
LOG.info("downloading kinesis-mock binary from %s", download_url)
download(download_url, bin_file_path)
chmod_r(bin_file_path, 0o777)
return bin_file_path
def install_elasticsearch():
if not os.path.exists(INSTALL_DIR_ES):
LOGGER.info('Downloading and installing local Elasticsearch server. This may take some time.')
mkdir(INSTALL_DIR_INFRA)
# download and extract archive
download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, TMP_ARCHIVE_ES, INSTALL_DIR_INFRA)
run('cd %s && mv elasticsearch* elasticsearch' % (INSTALL_DIR_INFRA))
for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):
dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
def install_elasticmq():
if SQS_BACKEND_IMPL != 'elasticmq':
return
# TODO remove this function if we stop using ElasticMQ entirely
if not os.path.exists(INSTALL_PATH_ELASTICMQ_JAR):
log_install_msg('ElasticMQ')
mkdir(INSTALL_DIR_ELASTICMQ)
# download archive
tmp_archive = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar')
if not os.path.exists(tmp_archive):
download(ELASTICMQ_JAR_URL, tmp_archive)
shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def start_kinesis(port=PORT_KINESIS, asynchronous=False, shard_limit=100, update_listener=None):
install.install_kinesalite()
backend_port = DEFAULT_PORT_KINESIS_BACKEND
kinesis_data_dir_param = ''
if DATA_DIR:
kinesis_data_dir = '%s/kinesis' % DATA_DIR
mkdir(kinesis_data_dir)
kinesis_data_dir_param = '--path %s' % kinesis_data_dir
cmd = ('%s/node_modules/kinesalite/cli.js --shardLimit %s --port %s %s' %
(ROOT_PATH, shard_limit, backend_port, kinesis_data_dir_param))
print('Starting mock Kinesis (%s port %s)...' % (get_service_protocol(), port))
do_run(cmd, asynchronous)
start_proxy_for_service('kinesis', port, backend_port, update_listener)
def start_dynamodb(port=PORT_DYNAMODB, asynchronous=False, update_listener=None):
install.install_dynamodb_local()
backend_port = DEFAULT_PORT_DYNAMODB_BACKEND
ddb_data_dir_param = '-inMemory'
if DATA_DIR:
ddb_data_dir = '%s/dynamodb' % DATA_DIR
mkdir(ddb_data_dir)
ddb_data_dir_param = '-dbPath %s' % ddb_data_dir
cmd = ('cd %s/infra/dynamodb/; java -Djava.library.path=./DynamoDBLocal_lib ' +
'-jar DynamoDBLocal.jar -sharedDb -port %s %s') % (ROOT_PATH, backend_port, ddb_data_dir_param)
print('Starting mock DynamoDB (%s port %s)...' % (get_service_protocol(), port))
start_proxy_for_service('dynamodb', port, backend_port, update_listener)
return do_run(cmd, asynchronous)
def install_elasticsearch():
if not os.path.exists(INSTALL_DIR_ES):
log_install_msg('Elasticsearch')
mkdir(INSTALL_DIR_INFRA)
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), 'localstack.es.zip')
download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, tmp_archive,
INSTALL_DIR_INFRA)
elasticsearch_dir = glob.glob(
os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*'))
if not elasticsearch_dir:
raise Exception('Unable to find Elasticsearch folder in %s' %
INSTALL_DIR_INFRA)
shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES)
for dir_name in ('data', 'logs', 'modules', 'plugins',
'config/scripts'):
dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
if is_alpine():
# https://github.com/pires/docker-elasticsearch/issues/56
os.environ['ES_TMPDIR'] = '/tmp'
plugin_binary = os.path.join(INSTALL_DIR_ES, 'bin',
'elasticsearch-plugin')
print('install elasticsearch-plugin %s' % (plugin))
run('%s install -b %s' % (plugin_binary, plugin))
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(INSTALL_DIR_ES, 'modules', plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(INSTALL_DIR_ES, 'modules', 'x-pack-ml',
'platform')
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(INSTALL_DIR_ES, 'config', 'jvm.options')
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)',
r'# \1',
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def install_elasticsearch(version=None):
version = get_elasticsearch_install_version(version)
install_dir = get_elasticsearch_install_dir(version)
installed_executable = os.path.join(install_dir, 'bin', 'elasticsearch')
if not os.path.exists(installed_executable):
log_install_msg('Elasticsearch (%s)' % version)
es_url = ELASTICSEARCH_URLS.get(version)
if not es_url:
raise Exception('Unable to find download URL for Elasticsearch version "%s"' % version)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(config.TMP_FOLDER, 'localstack.%s' % os.path.basename(es_url))
download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent)
elasticsearch_dir = glob.glob(os.path.join(install_dir_parent, 'elasticsearch*'))
if not elasticsearch_dir:
raise Exception('Unable to find Elasticsearch folder in %s' % install_dir_parent)
shutil.move(elasticsearch_dir[0], install_dir)
for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
if is_alpine():
# https://github.com/pires/docker-elasticsearch/issues/56
os.environ['ES_TMPDIR'] = '/tmp'
plugin_binary = os.path.join(install_dir, 'bin', 'elasticsearch-plugin')
plugin_dir = os.path.join(install_dir, 'plugins', plugin)
if not os.path.exists(plugin_dir):
LOG.info('Installing Elasticsearch plugin %s' % (plugin))
run('%s install -b %s' % (plugin_binary, plugin))
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(install_dir, 'modules', plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(install_dir, 'modules', 'x-pack-ml', 'platform')
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, 'config', 'jvm.options')
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)', r'# \1', jvm_options, flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None):
kinesis_mock_bin = install.install_kinesis_mock()
port = port or config.PORT_KINESIS
backend_port = get_free_tcp_port()
kinesis_data_dir_param = ''
if config.DATA_DIR:
kinesis_data_dir = '%s/kinesis' % config.DATA_DIR
mkdir(kinesis_data_dir)
# FIXME: workaround for https://github.com/localstack/localstack/issues/4227
streams_file = os.path.join(kinesis_data_dir, 'kinesis-data.json')
if not os.path.exists(streams_file):
with open(streams_file, 'w') as fd:
fd.write('{"streams":{}}')
kinesis_data_dir_param = 'SHOULD_PERSIST_DATA=true PERSIST_PATH=%s' % kinesis_data_dir
if not config.LS_LOG:
log_level = 'INFO'
elif config.LS_LOG == 'warning':
log_level = 'WARN'
else:
log_level = config.LS_LOG.upper()
log_level_param = 'LOG_LEVEL=%s' % log_level
latency = config.KINESIS_LATENCY + 'ms'
latency_param = 'CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s ' \
'START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s ' \
'DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s ' \
'UPDATE_SHARD_COUNT_DURATION=%s' \
% (latency, latency, latency, latency, latency, latency, latency, latency, latency)
if config.KINESIS_INITIALIZE_STREAMS != '':
initialize_streams_param = 'INITIALIZE_STREAMS=%s' % (
config.KINESIS_INITIALIZE_STREAMS)
else:
initialize_streams_param = ''
if kinesis_mock_bin.endswith('.jar'):
cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s' \
% (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param,
log_level_param, initialize_streams_param, kinesis_mock_bin)
else:
chmod_r(kinesis_mock_bin, 0o777)
cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1' \
% (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param,
log_level_param, initialize_streams_param, kinesis_mock_bin)
LOGGER.info('starting kinesis-mock proxy %d:%d with cmd: %s', port,
backend_port, cmd)
start_proxy_for_service('kinesis', port, backend_port, update_listener)
return do_run(cmd, asynchronous)
def install_amazon_kinesis_client_libs():
# install KCL/STS JAR files
if not os.path.exists(INSTALL_DIR_KCL):
mkdir(INSTALL_DIR_KCL)
if not os.path.exists(TMP_ARCHIVE_STS):
download(STS_JAR_URL, TMP_ARCHIVE_STS)
shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL)
# Compile Java files
from localstack.utils.kinesis import kclipy_helper
classpath = kclipy_helper.get_kcl_classpath()
java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH
class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH
if not glob.glob(class_files):
run('javac -cp "%s" %s' % (classpath, java_files))
def create_lambda_archive(script,
get_content=False,
libs=[],
runtime=None,
file_name=None):
"""Utility method to create a Lambda function archive"""
runtime = runtime or LAMBDA_DEFAULT_RUNTIME
tmp_dir = tempfile.mkdtemp(prefix=ARCHIVE_DIR_PREFIX)
TMP_FILES.append(tmp_dir)
file_name = file_name or get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER,
runtime=runtime)
script_file = os.path.join(tmp_dir, file_name)
if os.path.sep in script_file:
mkdir(os.path.dirname(script_file))
# create __init__.py files along the path to allow Python imports
path = file_name.split(os.path.sep)
for i in range(1, len(path)):
save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "")
save_file(script_file, script)
chmod_r(script_file, 0o777)
# copy libs
for lib in libs:
paths = [lib, "%s.py" % lib]
try:
module = importlib.import_module(lib)
paths.append(module.__file__)
except Exception:
pass
target_dir = tmp_dir
root_folder = os.path.join(LOCALSTACK_VENV_FOLDER,
"lib/python*/site-packages")
if lib == "localstack":
paths = ["localstack/*.py", "localstack/utils"]
root_folder = LOCALSTACK_ROOT_FOLDER
target_dir = os.path.join(tmp_dir, lib)
mkdir(target_dir)
for path in paths:
file_to_copy = path if path.startswith("/") else os.path.join(
root_folder, path)
for file_path in glob.glob(file_to_copy):
name = os.path.join(target_dir,
file_path.split(os.path.sep)[-1])
if os.path.isdir(file_path):
copy_dir(file_path, name)
else:
shutil.copyfile(file_path, name)
# create zip file
result = create_zip_file(tmp_dir, get_content=get_content)
return result
def install_amazon_kinesis_client_libs():
# install KCL/STS JAR files
if not os.path.exists(INSTALL_DIR_KCL):
mkdir(INSTALL_DIR_KCL)
tmp_archive = os.path.join(tempfile.gettempdir(),
'aws-java-sdk-sts.jar')
if not os.path.exists(tmp_archive):
download(STS_JAR_URL, tmp_archive)
shutil.copy(tmp_archive, INSTALL_DIR_KCL)
# Compile Java files
from localstack.utils.kinesis import kclipy_helper
classpath = kclipy_helper.get_kcl_classpath()
java_files = '%s/utils/kinesis/java/cloud/localstack/*.java' % ROOT_PATH
class_files = '%s/utils/kinesis/java/cloud/localstack/*.class' % ROOT_PATH
def get_lambda_code(func_name, retries=1, cache_time=None, env=None):
if MOCK_OBJ:
return ''
env = aws_stack.get_environment(env)
if cache_time is None and not aws_stack.is_local_env(env):
cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT
out = cmd_lambda('get-function --function-name %s' % func_name, env,
cache_time)
out = json.loads(out)
loc = out['Code']['Location']
hash = md5(loc)
folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash)
filename = 'archive.zip'
archive = '%s/%s' % (folder, filename)
try:
mkdir(folder)
if not os.path.isfile(archive):
download(loc, archive, verify_ssl=False)
if len(os.listdir(folder)) <= 1:
zip_path = os.path.join(folder, filename)
unzip(zip_path, folder)
except Exception as e:
print('WARN: %s' % e)
rm_rf(archive)
if retries > 0:
return get_lambda_code(func_name,
retries=retries - 1,
cache_time=1,
env=env)
else:
print('WARNING: Unable to retrieve lambda code: %s' % e)
# traverse subdirectories and get script sources
result = {}
for root, subdirs, files in os.walk(folder):
for file in files:
prefix = root.split(folder)[-1]
key = '%s/%s' % (prefix, file)
if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key):
codefile = '%s/%s' % (root, file)
result[key] = load_file(codefile)
# cleanup cache
clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN,
last_clean_time=last_cache_cleanup_time,
max_age=TMP_DOWNLOAD_CACHE_MAX_AGE)
# TODO: delete only if cache_time is over
rm_rf(folder)
return result
def install_amazon_kinesis_client_libs():
# install KCL/STS JAR files
if not os.path.exists(INSTALL_DIR_KCL):
mkdir(INSTALL_DIR_KCL)
if not os.path.exists(TMP_ARCHIVE_STS):
download(STS_JAR_URL, TMP_ARCHIVE_STS)
shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL)
# Compile Java files
from localstack.utils.kinesis import kclipy_helper
classpath = kclipy_helper.get_kcl_classpath()
java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH
class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH
if not glob.glob(class_files):
run('javac -cp "%s" %s' % (classpath, java_files))
def get_lambda_code(func_name, retries=1, cache_time=None, env=None):
if MOCK_OBJ:
return ''
env = aws_stack.get_environment(env)
if cache_time is None and env.region != REGION_LOCAL:
cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT
out = cmd_lambda('get-function --function-name %s' % func_name, env, cache_time)
out = json.loads(out)
loc = out['Code']['Location']
hash = md5(loc)
folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash)
filename = 'archive.zip'
archive = '%s/%s' % (folder, filename)
try:
mkdir(folder)
if not os.path.isfile(archive):
download(loc, archive, verify_ssl=False)
if len(os.listdir(folder)) <= 1:
zip_path = os.path.join(folder, filename)
unzip(zip_path, folder)
except Exception as e:
print('WARN: %s' % e)
rm_rf(archive)
if retries > 0:
return get_lambda_code(func_name, retries=retries - 1, cache_time=1, env=env)
else:
print('WARNING: Unable to retrieve lambda code: %s' % e)
# traverse subdirectories and get script sources
result = {}
for root, subdirs, files in os.walk(folder):
for file in files:
prefix = root.split(folder)[-1]
key = '%s/%s' % (prefix, file)
if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key):
codefile = '%s/%s' % (root, file)
result[key] = load_file(codefile)
# cleanup cache
clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN,
last_clean_time=last_cache_cleanup_time,
max_age=TMP_DOWNLOAD_CACHE_MAX_AGE)
# TODO: delete only if cache_time is over
rm_rf(folder)
return result
def create_lambda_archive(script, stream=None, get_content=False, libs=[], runtime=None):
"""Utility method to create a Lambda function archive"""
tmp_dir = tempfile.mkdtemp(prefix=ARCHIVE_DIR_PREFIX)
TMP_FILES.append(tmp_dir)
file_name = get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER, runtime=runtime)
script_file = '%s/%s' % (tmp_dir, file_name)
save_file(script_file, script)
# copy libs
for lib in libs:
paths = [lib, '%s.py' % lib]
target_dir = tmp_dir
root_folder = '%s/lib/python*/site-packages' % LOCALSTACK_VENV_FOLDER
if lib == 'localstack':
paths = ['localstack/*.py', 'localstack/utils']
root_folder = LOCALSTACK_ROOT_FOLDER
target_dir = '%s/%s/' % (tmp_dir, lib)
mkdir(target_dir)
for path in paths:
file_to_copy = '%s/%s' % (root_folder, path)
for file_path in glob.glob(file_to_copy):
run('cp -r %s %s/' % (file_path, target_dir))
# create zip file
return create_zip_file(tmp_dir, get_content=True)
def test_lambda_runtimes():
lambda_client = aws_stack.connect_to_service('lambda')
# deploy and invoke lambda - Python 2.7
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY,
zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
if use_docker():
# deploy and invoke lambda - Python 3.6
zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON3), get_content=True,
libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3,
zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke lambda - Java
if not os.path.exists(TEST_LAMBDA_JAVA):
mkdir(os.path.dirname(TEST_LAMBDA_JAVA))
download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA)
zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA, zip_file=zip_file,
runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert 'LinkedHashMap' in to_str(result_data)
# test SNSEvent
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event',
Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {'async': 'True'}
# test DDBEvent
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event',
Payload=b'{"Records": [{"dynamodb": {"Message": "{}"}}]}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {'async': 'True'}
# test KinesisEvent
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA,
Payload=b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert 'KinesisEvent' in to_str(result_data)
# deploy and invoke lambda - Java with stream handler
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=zip_file,
runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke lambda - Java with serializable input object
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, zip_file=zip_file,
runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.SerializedInputLambdaHandler')
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE,
Payload=b'{"bucket": "test_bucket", "key": "test_key"}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert json.loads(to_str(result_data)) == {'validated': True, 'bucket': 'test_bucket', 'key': 'test_key'}
if use_docker():
# deploy and invoke lambda - Node.js
zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True)
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS,
zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
# deploy and invoke - .NET Core 2.0. Its already a zip
zip_file = TEST_LAMBDA_DOTNETCORE2
zip_file_content = None
with open(zip_file, 'rb') as file_obj:
zip_file_content = file_obj.read()
testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DOTNETCORE2, zip_file=zip_file_content,
handler='DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler',
runtime=LAMBDA_RUNTIME_DOTNETCORE2)
result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}')
assert result['StatusCode'] == 200
result_data = result['Payload'].read()
assert to_str(result_data).strip() == '{}'
def check_dynamodb(expect_shutdown=False, print_error=False):
out = None
try:
# wait for port to be opened
wait_for_port_open(DEFAULT_PORT_DYNAMODB_BACKEND)
# check DynamoDB
out = aws_stack.connect_to_service(service_name='dynamodb').list_tables()
except Exception as e:
if print_error:
LOGGER.error('DynamoDB health check failed: %s %s' % (e, traceback.format_exc()))
if expect_shutdown:
assert out is None
else:
assert isinstance(out['TableNames'], list)
def start_dynamodb(port=PORT_DYNAMODB, async=False, update_listener=None):
install.install_dynamodb_local()
backend_port = DEFAULT_PORT_DYNAMODB_BACKEND
ddb_data_dir_param = '-inMemory'
if DATA_DIR:
ddb_data_dir = '%s/dynamodb' % DATA_DIR
mkdir(ddb_data_dir)
ddb_data_dir_param = '-dbPath %s' % ddb_data_dir
cmd = ('cd %s/infra/dynamodb/; java -Djava.library.path=./DynamoDBLocal_lib ' +
'-jar DynamoDBLocal.jar -sharedDb -port %s %s') % (ROOT_PATH, backend_port, ddb_data_dir_param)
print('Starting mock DynamoDB (%s port %s)...' % (get_service_protocol(), port))
start_proxy_for_service('dynamodb', port, backend_port, update_listener)
return do_run(cmd, async)
def start_infra_in_docker():
# load plugins before starting the docker container
plugin_configs = load_plugins()
plugin_run_params = ' '.join([
entry.get('docker', {}).get('run_flags', '') for entry in plugin_configs])
services = os.environ.get('SERVICES', '')
entrypoint = os.environ.get('ENTRYPOINT', '')
cmd = os.environ.get('CMD', '')
image_name = os.environ.get('IMAGE_NAME', constants.DOCKER_IMAGE_NAME)
service_ports = config.SERVICE_PORTS
force_noninteractive = os.environ.get('FORCE_NONINTERACTIVE', '')
# construct port mappings
ports_list = sorted(service_ports.values())
start_port = 0
last_port = 0
port_ranges = []
for i in range(0, len(ports_list)):
if not start_port:
start_port = ports_list[i]
if not last_port:
last_port = ports_list[i]
if ports_list[i] > last_port + 1:
port_ranges.append([start_port, last_port])
start_port = ports_list[i]
elif i >= len(ports_list) - 1:
port_ranges.append([start_port, ports_list[i]])
last_port = ports_list[i]
port_mappings = ' '.join(
'-p {start}-{end}:{start}-{end}'.format(start=entry[0], end=entry[1])
if entry[0] < entry[1] else '-p {port}:{port}'.format(port=entry[0])
for entry in port_ranges)
if services:
port_mappings = ''
for service, port in service_ports.items():
port_mappings += ' -p {port}:{port}'.format(port=port)
env_str = ''
for env_var in config.CONFIG_ENV_VARS:
value = os.environ.get(env_var, None)
if value is not None:
env_str += '-e %s="%s" ' % (env_var, value)
data_dir_mount = ''
data_dir = os.environ.get('DATA_DIR', None)
if data_dir is not None:
container_data_dir = '/tmp/localstack_data'
data_dir_mount = '-v "%s:%s" ' % (data_dir, container_data_dir)
env_str += '-e DATA_DIR="%s" ' % container_data_dir
interactive = '' if force_noninteractive or in_ci() else '-it '
# append space if parameter is set
entrypoint = '%s ' % entrypoint if entrypoint else entrypoint
plugin_run_params = '%s ' % plugin_run_params if plugin_run_params else plugin_run_params
docker_cmd = ('docker run %s%s%s%s' +
'-p 8080:8080 %s %s' +
'-v "%s:/tmp/localstack" -v "%s:%s" ' +
'-e DOCKER_HOST="unix://%s" ' +
'-e HOST_TMP_FOLDER="%s" "%s" %s') % (
interactive, entrypoint, env_str, plugin_run_params, port_mappings, data_dir_mount,
config.TMP_FOLDER, config.DOCKER_SOCK, config.DOCKER_SOCK, config.DOCKER_SOCK,
config.HOST_TMP_FOLDER, image_name, cmd
)
mkdir(config.TMP_FOLDER)
run_cmd_safe(cmd='chmod -R 777 "%s"' % config.TMP_FOLDER)
print(docker_cmd)
t = ShellCommandThread(docker_cmd, outfile=subprocess.PIPE)
t.start()
time.sleep(2)
t.process.wait()
sys.exit(t.process.returncode)
def set_function_code(code, lambda_name):
def generic_handler(event, context):
raise Exception(('Unable to find executor for Lambda function "%s". ' +
'Note that Node.js and .NET Core Lambdas currently require LAMBDA_EXECUTOR=docker') % lambda_name)
lambda_handler = generic_handler
lambda_cwd = None
arn = func_arn(lambda_name)
runtime = arn_to_lambda[arn].runtime
handler_name = arn_to_lambda.get(arn).handler
lambda_environment = arn_to_lambda.get(arn).envvars
if not handler_name:
handler_name = LAMBDA_DEFAULT_HANDLER
handler_file = get_handler_file_from_name(handler_name, runtime=runtime)
handler_function = get_handler_function_from_name(handler_name, runtime=runtime)
# Stop/remove any containers that this arn uses.
LAMBDA_EXECUTOR.cleanup(arn)
if 'S3Bucket' in code:
s3_client = aws_stack.connect_to_service('s3')
bytes_io = BytesIO()
try:
s3_client.download_fileobj(code['S3Bucket'], code['S3Key'], bytes_io)
zip_file_content = bytes_io.getvalue()
except Exception as e:
return error_response('Unable to fetch Lambda archive from S3: %s' % e, 404)
elif 'ZipFile' in code:
zip_file_content = code['ZipFile']
zip_file_content = base64.b64decode(zip_file_content)
else:
return error_response('No valid Lambda archive specified.', 400)
# save tmp file
tmp_dir = '%s/zipfile.%s' % (config.TMP_FOLDER, short_uid())
mkdir(tmp_dir)
tmp_file = '%s/%s' % (tmp_dir, LAMBDA_ZIP_FILE_NAME)
save_file(tmp_file, zip_file_content)
TMP_FILES.append(tmp_dir)
lambda_cwd = tmp_dir
# check if this is a ZIP file
is_zip = is_zip_file(zip_file_content)
if is_zip:
unzip(tmp_file, tmp_dir)
main_file = '%s/%s' % (tmp_dir, handler_file)
if not os.path.isfile(main_file):
# check if this is a zip file that contains a single JAR file
jar_files = glob.glob('%s/*.jar' % tmp_dir)
if len(jar_files) == 1:
main_file = jar_files[0]
if os.path.isfile(main_file):
# make sure the file is actually readable, then read contents
ensure_readable(main_file)
with open(main_file, 'rb') as file_obj:
zip_file_content = file_obj.read()
else:
file_list = run('ls -la %s' % tmp_dir)
LOG.debug('Lambda archive content:\n%s' % file_list)
return error_response('Unable to find handler script in Lambda archive.', 400, error_type='ValidationError')
# it could be a JAR file (regardless of whether wrapped in a ZIP file or not)
is_jar = is_jar_archive(zip_file_content)
if is_jar:
def execute(event, context):
result, log_output = lambda_executors.EXECUTOR_LOCAL.execute_java_lambda(event, context,
handler=arn_to_lambda[arn].handler, main_file=main_file)
return result
lambda_handler = execute
elif runtime.startswith('python') and not use_docker():
try:
lambda_handler = exec_lambda_code(zip_file_content,
handler_function=handler_function, lambda_cwd=lambda_cwd,
lambda_env=lambda_environment)
except Exception as e:
raise Exception('Unable to get handler function from lambda code.', e)
if not is_zip and not is_jar:
raise Exception('Uploaded Lambda code is neither a ZIP nor JAR file.')
add_function_mapping(lambda_name, lambda_handler, lambda_cwd)
return {'FunctionName': lambda_name}
from localstack.utils.aws import aws_stack
from localstack.utils.common import mkdir
from localstack.services import install
from localstack.services.infra import get_service_protocol, start_proxy_for_service, do_run
from localstack.services.install import ROOT_PATH
LOGGER = logging.getLogger(__name__)
def start_kinesis(port=PORT_KINESIS, async=False, shard_limit=100, update_listener=None):
install.install_kinesalite()
backend_port = DEFAULT_PORT_KINESIS_BACKEND
kinesis_data_dir_param = ''
if DATA_DIR:
kinesis_data_dir = '%s/kinesis' % DATA_DIR
mkdir(kinesis_data_dir)
kinesis_data_dir_param = '--path %s' % kinesis_data_dir
cmd = ('%s/node_modules/kinesalite/cli.js --shardLimit %s --port %s %s' %
(ROOT_PATH, shard_limit, backend_port, kinesis_data_dir_param))
print('Starting mock Kinesis (%s port %s)...' % (get_service_protocol(), port))
start_proxy_for_service('kinesis', port, backend_port, update_listener)
return do_run(cmd, async)
def check_kinesis(expect_shutdown=False, print_error=False):
out = None
try:
# check Kinesis
out = aws_stack.connect_to_service(service_name='kinesis').list_streams()
except Exception as e:
if print_error:
