def install_kinesis_mock(bin_file_path: str = None):
response = requests.get(KINESIS_MOCK_RELEASE_URL)
if not response.ok:
raise ValueError(
f"Could not get list of releases from {KINESIS_MOCK_RELEASE_URL}: {response.text}"
)
bin_file_path = bin_file_path or kinesis_mock_install_path()
github_release = response.json()
download_url = None
bin_file_name = os.path.basename(bin_file_path)
for asset in github_release.get("assets", []):
# find the correct binary in the release
if asset["name"] == bin_file_name:
download_url = asset["browser_download_url"]
break
if download_url is None:
raise ValueError(
f"Could not find required binary {bin_file_name} in release {KINESIS_MOCK_RELEASE_URL}"
)
mkdir(INSTALL_DIR_KINESIS_MOCK)
LOG.info("downloading kinesis-mock binary from %s", download_url)
download(download_url, bin_file_path)
chmod_r(bin_file_path, 0o777)
def download_and_extract(archive_url,
target_dir,
retries=0,
sleep=3,
tmp_archive=None):
mkdir(target_dir)
_, ext = os.path.splitext(tmp_archive or archive_url)
tmp_archive = tmp_archive or new_tmp_file()
if not os.path.exists(tmp_archive) or os.path.getsize(tmp_archive) <= 0:
# create temporary placeholder file, to avoid duplicate parallel downloads
save_file(tmp_archive, "")
for i in range(retries + 1):
try:
download(archive_url, tmp_archive)
break
except Exception:
time.sleep(sleep)
if ext == ".zip":
unzip(tmp_archive, target_dir)
elif ext in [".bz2", ".gz", ".tgz"]:
untar(tmp_archive, target_dir)
else:
raise Exception(f"Unsupported archive format: {ext}")
def save_for_retrospection(id: str, region: str, **kwargs: Dict[str, Any]):
"""Save a message for retrospection.
The email is saved to filesystem and is also made accessible via a service endpoint.
kwargs should consist of following keys related to the email:
- Body
- Destinations
- RawData
- Source
- Subject
- Template
- TemplateData
"""
ses_dir = os.path.join(config.dirs.data or config.dirs.tmp, "ses")
mkdir(ses_dir)
path = os.path.join(ses_dir, id + ".json")
email = {"Id": id, "Region": region, **kwargs}
EMAILS[id] = email
def _serialize(obj):
"""JSON serializer for timestamps."""
if isinstance(obj, (datetime, date, time)):
return obj.isoformat()
return obj.__dict__
with open(path, "w") as f:
f.write(json.dumps(email, default=_serialize))
LOGGER.debug("Email saved at: %s", path)
def install_local_kms():
local_arch = f"{platform.system().lower()}-{get_arch()}"
binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", local_arch)
if not os.path.exists(binary_path):
log_install_msg("KMS")
mkdir(INSTALL_DIR_KMS)
kms_url = KMS_URL_PATTERN.replace("<arch>", local_arch)
download(kms_url, binary_path)
chmod_r(binary_path, 0o777)
def install_elasticmq():
# TODO remove this function if we stop using ElasticMQ entirely
if not os.path.exists(INSTALL_PATH_ELASTICMQ_JAR):
log_install_msg("ElasticMQ")
mkdir(INSTALL_DIR_ELASTICMQ)
# download archive
tmp_archive = os.path.join(config.dirs.tmp, "elasticmq-server.jar")
if not os.path.exists(tmp_archive):
download(ELASTICMQ_JAR_URL, tmp_archive)
shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def install_opensearch(version=None):
# locally import to avoid having a dependency on ASF when starting the CLI
from localstack.aws.api.opensearch import EngineType
from localstack.services.opensearch import versions
if not version:
version = OPENSEARCH_DEFAULT_VERSION
version = get_opensearch_install_version(version)
install_dir = get_opensearch_install_dir(version)
installed_executable = os.path.join(install_dir, "bin", "opensearch")
if not os.path.exists(installed_executable):
with OS_INSTALL_LOCKS.setdefault(version, threading.Lock()):
if not os.path.exists(installed_executable):
log_install_msg(f"OpenSearch ({version})")
opensearch_url = versions.get_download_url(
version, EngineType.OpenSearch)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(
config.dirs.tmp,
f"localstack.{os.path.basename(opensearch_url)}")
download_and_extract_with_retry(opensearch_url, tmp_archive,
install_dir_parent)
opensearch_dir = glob.glob(
os.path.join(install_dir_parent, "opensearch*"))
if not opensearch_dir:
raise Exception(
f"Unable to find OpenSearch folder in {install_dir_parent}"
)
shutil.move(opensearch_dir[0], install_dir)
for dir_name in ("data", "logs", "modules", "plugins",
"config/scripts"):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, "config", "jvm.options")
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)",
r"# \1",
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def install_amazon_kinesis_client_libs():
# install KCL/STS JAR files
if not os.path.exists(INSTALL_PATH_KCL_JAR):
mkdir(INSTALL_DIR_KCL)
tmp_archive = os.path.join(tempfile.gettempdir(),
"aws-java-sdk-sts.jar")
if not os.path.exists(tmp_archive):
download(STS_JAR_URL, tmp_archive)
shutil.copy(tmp_archive, INSTALL_DIR_KCL)
# Compile Java files
from localstack.utils.kinesis import kclipy_helper
classpath = kclipy_helper.get_kcl_classpath()
if is_windows():
classpath = re.sub(r":([^\\])", r";\1", classpath)
java_files = f"{MODULE_MAIN_PATH}/utils/kinesis/java/cloud/localstack/*.java"
class_files = f"{MODULE_MAIN_PATH}/utils/kinesis/java/cloud/localstack/*.class"
if not glob.glob(class_files):
run(f'javac -source {JAVAC_TARGET_VERSION} -target {JAVAC_TARGET_VERSION} -cp "{classpath}" {java_files}'
)
def prepare_docker_start():
# prepare environment for docker start
container_name = config.MAIN_CONTAINER_NAME
if DOCKER_CLIENT.is_container_running(container_name):
raise ContainerExists(
'LocalStack container named "%s" is already running' %
container_name)
if config.dirs.tmp != config.dirs.functions and not config.LAMBDA_REMOTE_DOCKER:
# Logger is not initialized at this point, so the warning is displayed via print
print(
f"WARNING: The detected temp folder for localstack ({config.dirs.tmp}) is not equal to the "
f"HOST_TMP_FOLDER environment variable set ({config.dirs.functions})."
)
os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1"
# make sure temp folder exists
mkdir(config.dirs.tmp)
try:
chmod_r(config.dirs.tmp, 0o777)
except Exception:
pass
def create_lambda_archive(
script: str,
get_content: bool = False,
libs: List[str] = None,
runtime: str = None,
file_name: str = None,
exclude_func: Callable[[str], bool] = None,
):
"""Utility method to create a Lambda function archive"""
if libs is None:
libs = []
runtime = runtime or LAMBDA_DEFAULT_RUNTIME
with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir:
file_name = file_name or get_handler_file_from_name(
LAMBDA_DEFAULT_HANDLER, runtime=runtime)
script_file = os.path.join(tmp_dir, file_name)
if os.path.sep in script_file:
mkdir(os.path.dirname(script_file))
# create __init__.py files along the path to allow Python imports
path = file_name.split(os.path.sep)
for i in range(1, len(path)):
save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])),
"")
save_file(script_file, script)
chmod_r(script_file, 0o777)
# copy libs
for lib in libs:
paths = [lib, "%s.py" % lib]
try:
module = importlib.import_module(lib)
paths.append(module.__file__)
except Exception:
pass
target_dir = tmp_dir
root_folder = os.path.join(LOCALSTACK_VENV_FOLDER,
"lib/python*/site-packages")
if lib == "localstack":
paths = ["localstack/*.py", "localstack/utils"]
root_folder = LOCALSTACK_ROOT_FOLDER
target_dir = os.path.join(tmp_dir, lib)
mkdir(target_dir)
for path in paths:
file_to_copy = path if path.startswith("/") else os.path.join(
root_folder, path)
for file_path in glob.glob(file_to_copy):
name = os.path.join(target_dir,
file_path.split(os.path.sep)[-1])
if os.path.isdir(file_path):
copy_dir(file_path, name)
else:
shutil.copyfile(file_path, name)
if exclude_func:
for dirpath, folders, files in os.walk(tmp_dir):
for name in list(folders) + list(files):
full_name = os.path.join(dirpath, name)
relative = os.path.relpath(full_name, start=tmp_dir)
if exclude_func(relative):
rm_rf(full_name)
# create zip file
result = create_zip_file(tmp_dir, get_content=get_content)
return result
def install_lambda_java_testlibs():
# Download the LocalStack Utils Test jar file from the maven repo
if not os.path.exists(TEST_LAMBDA_JAVA):
mkdir(os.path.dirname(TEST_LAMBDA_JAVA))
download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA)
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
if not DOCKER_CLIENT.has_docker():
# TODO: works only when a docker socket is available -> add a fallback if running without Docker?
LOG.warning(
"Docker not available - skipping installation of StepFunctions dependency"
)
return
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
docker_name = "tmp-ls-sfn"
DOCKER_CLIENT.run_container(
IMAGE_NAME_SFN_LOCAL,
remove=True,
entrypoint="",
name=docker_name,
detach=True,
command=["sleep", "15"],
)
time.sleep(5)
DOCKER_CLIENT.copy_from_container(
docker_name,
local_path=dirs.static_libs,
container_path="/home/stepfunctionslocal/")
path = Path(f"{dirs.static_libs}/stepfunctionslocal/")
for file in path.glob("*.jar"):
file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
rm_rf(str(path))
classes = [
SFN_PATCH_CLASS1,
SFN_PATCH_CLASS2,
SFN_PATCH_CLASS_REGION,
SFN_PATCH_CLASS_STARTER,
SFN_PATCH_CLASS_ASYNC2SERVICEAPI,
SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED,
SFN_PATCH_FILE_METAINF,
]
for patch_class in classes:
patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}"
add_file_to_jar(patch_class,
patch_url,
target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR)
# special case for Manifest file - extract first, replace content, then update in JAR file
manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF",
"MANIFEST.MF")
if not os.path.exists(manifest_file):
content = run([
"unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR,
"META-INF/MANIFEST.MF"
])
content = re.sub("Main-Class: .+",
"Main-Class: cloud.localstack.StepFunctionsStarter",
content)
classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS])
content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ",
content)
save_file(manifest_file, content)
run(
["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"],
cwd=INSTALL_DIR_STEPFUNCTIONS,
)
# download additional jar libs
for jar_url in JAR_URLS:
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
os.path.basename(jar_url))
if not file_exists_not_empty(target):
download(jar_url, target)
# download aws-sdk lambda handler
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
"localstack-internal-awssdk", "awssdk.zip")
if not file_exists_not_empty(target):
download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def install_elasticsearch(version=None):
# locally import to avoid having a dependency on ASF when starting the CLI
from localstack.aws.api.opensearch import EngineType
from localstack.services.opensearch import versions
if not version:
version = ELASTICSEARCH_DEFAULT_VERSION
version = get_elasticsearch_install_version(version)
install_dir = get_elasticsearch_install_dir(version)
installed_executable = os.path.join(install_dir, "bin", "elasticsearch")
if not os.path.exists(installed_executable):
log_install_msg(f"Elasticsearch ({version})")
es_url = versions.get_download_url(version, EngineType.Elasticsearch)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(config.dirs.tmp,
f"localstack.{os.path.basename(es_url)}")
download_and_extract_with_retry(es_url, tmp_archive,
install_dir_parent)
elasticsearch_dir = glob.glob(
os.path.join(install_dir_parent, "elasticsearch*"))
if not elasticsearch_dir:
raise Exception(
f"Unable to find Elasticsearch folder in {install_dir_parent}")
shutil.move(elasticsearch_dir[0], install_dir)
for dir_name in ("data", "logs", "modules", "plugins",
"config/scripts"):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
plugin_binary = os.path.join(install_dir, "bin",
"elasticsearch-plugin")
plugin_dir = os.path.join(install_dir, "plugins", plugin)
if not os.path.exists(plugin_dir):
LOG.info("Installing Elasticsearch plugin %s", plugin)
def try_install():
output = run([plugin_binary, "install", "-b", plugin])
LOG.debug("Plugin installation output: %s", output)
# We're occasionally seeing javax.net.ssl.SSLHandshakeException -> add download retries
download_attempts = 3
try:
retry(try_install, retries=download_attempts - 1, sleep=2)
except Exception:
LOG.warning(
"Unable to download Elasticsearch plugin '%s' after %s attempts",
plugin,
download_attempts,
)
if not os.environ.get("IGNORE_ES_DOWNLOAD_ERRORS"):
raise
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(install_dir, "modules", plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(install_dir, "modules", "x-pack-ml", "platform")
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, "config", "jvm.options")
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)",
r"# \1",
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
