def download_and_extract_with_retry(archive_url, tmp_archive, target_dir):
try:
download_and_extract(archive_url, target_dir, tmp_archive=tmp_archive)
except Exception as e:
# try deleting and re-downloading the zip file
LOG.info("Unable to extract file, re-downloading ZIP archive %s: %s",
tmp_archive, e)
rm_rf(tmp_archive)
download_and_extract(archive_url, target_dir, tmp_archive=tmp_archive)
def create_dynamodb_server(port=None,
db_path: Optional[str] = None,
clean_db_path: bool = False) -> DynamodbServer:
"""
Creates a dynamodb server from the LocalStack configuration.
"""
port = port or get_free_tcp_port()
server = DynamodbServer(port)
db_path = f"{config.dirs.data}/dynamodb" if not db_path and config.dirs.data else db_path
if db_path:
if clean_db_path:
rm_rf(db_path)
mkdir(db_path)
absolute_path = os.path.abspath(db_path)
server.db_path = absolute_path
server.heap_size = config.DYNAMODB_HEAP_SIZE
server.share_db = is_env_true("DYNAMODB_SHARE_DB")
server.optimize_db_before_startup = is_env_true(
"DYNAMODB_OPTIMIZE_DB_BEFORE_STARTUP")
server.delay_transient_statuses = is_env_true(
"DYNAMODB_DELAY_TRANSIENT_STATUSES")
server.cors = os.getenv("DYNAMODB_CORS", None)
return server
def create_lambda_archive(
script: str,
get_content: bool = False,
libs: List[str] = None,
runtime: str = None,
file_name: str = None,
exclude_func: Callable[[str], bool] = None,
):
"""Utility method to create a Lambda function archive"""
if libs is None:
libs = []
runtime = runtime or LAMBDA_DEFAULT_RUNTIME
with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir:
file_name = file_name or get_handler_file_from_name(
LAMBDA_DEFAULT_HANDLER, runtime=runtime)
script_file = os.path.join(tmp_dir, file_name)
if os.path.sep in script_file:
mkdir(os.path.dirname(script_file))
# create __init__.py files along the path to allow Python imports
path = file_name.split(os.path.sep)
for i in range(1, len(path)):
save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])),
"")
save_file(script_file, script)
chmod_r(script_file, 0o777)
# copy libs
for lib in libs:
paths = [lib, "%s.py" % lib]
try:
module = importlib.import_module(lib)
paths.append(module.__file__)
except Exception:
pass
target_dir = tmp_dir
root_folder = os.path.join(LOCALSTACK_VENV_FOLDER,
"lib/python*/site-packages")
if lib == "localstack":
paths = ["localstack/*.py", "localstack/utils"]
root_folder = LOCALSTACK_ROOT_FOLDER
target_dir = os.path.join(tmp_dir, lib)
mkdir(target_dir)
for path in paths:
file_to_copy = path if path.startswith("/") else os.path.join(
root_folder, path)
for file_path in glob.glob(file_to_copy):
name = os.path.join(target_dir,
file_path.split(os.path.sep)[-1])
if os.path.isdir(file_path):
copy_dir(file_path, name)
else:
shutil.copyfile(file_path, name)
if exclude_func:
for dirpath, folders, files in os.walk(tmp_dir):
for name in list(folders) + list(files):
full_name = os.path.join(dirpath, name)
relative = os.path.relpath(full_name, start=tmp_dir)
if exclude_func(relative):
rm_rf(full_name)
# create zip file
result = create_zip_file(tmp_dir, get_content=get_content)
return result
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
if not DOCKER_CLIENT.has_docker():
# TODO: works only when a docker socket is available -> add a fallback if running without Docker?
LOG.warning(
"Docker not available - skipping installation of StepFunctions dependency"
)
return
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
docker_name = "tmp-ls-sfn"
DOCKER_CLIENT.run_container(
IMAGE_NAME_SFN_LOCAL,
remove=True,
entrypoint="",
name=docker_name,
detach=True,
command=["sleep", "15"],
)
time.sleep(5)
DOCKER_CLIENT.copy_from_container(
docker_name,
local_path=dirs.static_libs,
container_path="/home/stepfunctionslocal/")
path = Path(f"{dirs.static_libs}/stepfunctionslocal/")
for file in path.glob("*.jar"):
file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
rm_rf(str(path))
classes = [
SFN_PATCH_CLASS1,
SFN_PATCH_CLASS2,
SFN_PATCH_CLASS_REGION,
SFN_PATCH_CLASS_STARTER,
SFN_PATCH_CLASS_ASYNC2SERVICEAPI,
SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED,
SFN_PATCH_FILE_METAINF,
]
for patch_class in classes:
patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}"
add_file_to_jar(patch_class,
patch_url,
target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR)
# special case for Manifest file - extract first, replace content, then update in JAR file
manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF",
"MANIFEST.MF")
if not os.path.exists(manifest_file):
content = run([
"unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR,
"META-INF/MANIFEST.MF"
])
content = re.sub("Main-Class: .+",
"Main-Class: cloud.localstack.StepFunctionsStarter",
content)
classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS])
content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ",
content)
save_file(manifest_file, content)
run(
["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"],
cwd=INSTALL_DIR_STEPFUNCTIONS,
)
# download additional jar libs
for jar_url in JAR_URLS:
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
os.path.basename(jar_url))
if not file_exists_not_empty(target):
download(jar_url, target)
# download aws-sdk lambda handler
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
"localstack-internal-awssdk", "awssdk.zip")
if not file_exists_not_empty(target):
download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def install_elasticsearch(version=None):
# locally import to avoid having a dependency on ASF when starting the CLI
from localstack.aws.api.opensearch import EngineType
from localstack.services.opensearch import versions
if not version:
version = ELASTICSEARCH_DEFAULT_VERSION
version = get_elasticsearch_install_version(version)
install_dir = get_elasticsearch_install_dir(version)
installed_executable = os.path.join(install_dir, "bin", "elasticsearch")
if not os.path.exists(installed_executable):
log_install_msg(f"Elasticsearch ({version})")
es_url = versions.get_download_url(version, EngineType.Elasticsearch)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(config.dirs.tmp,
f"localstack.{os.path.basename(es_url)}")
download_and_extract_with_retry(es_url, tmp_archive,
install_dir_parent)
elasticsearch_dir = glob.glob(
os.path.join(install_dir_parent, "elasticsearch*"))
if not elasticsearch_dir:
raise Exception(
f"Unable to find Elasticsearch folder in {install_dir_parent}")
shutil.move(elasticsearch_dir[0], install_dir)
for dir_name in ("data", "logs", "modules", "plugins",
"config/scripts"):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
plugin_binary = os.path.join(install_dir, "bin",
"elasticsearch-plugin")
plugin_dir = os.path.join(install_dir, "plugins", plugin)
if not os.path.exists(plugin_dir):
LOG.info("Installing Elasticsearch plugin %s", plugin)
def try_install():
output = run([plugin_binary, "install", "-b", plugin])
LOG.debug("Plugin installation output: %s", output)
# We're occasionally seeing javax.net.ssl.SSLHandshakeException -> add download retries
download_attempts = 3
try:
retry(try_install, retries=download_attempts - 1, sleep=2)
except Exception:
LOG.warning(
"Unable to download Elasticsearch plugin '%s' after %s attempts",
plugin,
download_attempts,
)
if not os.environ.get("IGNORE_ES_DOWNLOAD_ERRORS"):
raise
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(install_dir, "modules", plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(install_dir, "modules", "x-pack-ml", "platform")
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, "config", "jvm.options")
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)",
r"# \1",
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def listen_to_kinesis(
stream_name,
listener_func=None,
processor_script=None,
events_file=None,
endpoint_url=None,
log_file=None,
configs=None,
env=None,
ddb_lease_table_suffix=None,
env_vars=None,
kcl_log_level=DEFAULT_KCL_LOG_LEVEL,
log_subscribers=None,
wait_until_started=False,
fh_d_stream=None,
region_name=None,
):
"""
High-level function that allows to subscribe to a Kinesis stream
and receive events in a listener function. A KCL client process is
automatically started in the background.
"""
if configs is None:
configs = {}
if env_vars is None:
env_vars = {}
if log_subscribers is None:
log_subscribers = []
env = aws_stack.get_environment(env)
if not events_file:
events_file = EVENTS_FILE_PATTERN.replace("*", short_uid())
TMP_FILES.append(events_file)
if not processor_script:
processor_script = generate_processor_script(events_file,
log_file=log_file)
rm_rf(events_file)
# start event reader thread (this process)
ready_mutex = threading.Semaphore(0)
thread = EventFileReaderThread(events_file,
listener_func,
ready_mutex=ready_mutex,
fh_d_stream=fh_d_stream)
thread.start()
# Wait until the event reader thread is ready (to avoid 'Connection refused' error on the UNIX socket)
ready_mutex.acquire()
# start KCL client (background process)
if processor_script[-4:] == ".pyc":
processor_script = processor_script[0:-1]
# add log listener that notifies when KCL is started
if wait_until_started:
listener = KclStartedLogListener()
log_subscribers.append(listener)
process = start_kcl_client_process(
stream_name,
processor_script,
endpoint_url=endpoint_url,
log_file=log_file,
configs=configs,
env=env,
ddb_lease_table_suffix=ddb_lease_table_suffix,
env_vars=env_vars,
kcl_log_level=kcl_log_level,
log_subscribers=log_subscribers,
region_name=region_name,
)
if wait_until_started:
# Wait at most 90 seconds for initialization. Note that creating the DDB table can take quite a bit
try:
listener.sync_init.get(block=True, timeout=90)
except Exception:
raise Exception("Timeout when waiting for KCL initialization.")
# wait at most 30 seconds for shard lease notification
try:
listener.sync_take_shard.get(block=True, timeout=30)
except Exception:
# this merely means that there is no shard available to take. Do nothing.
pass
return process
def rm_env_vars_file(env_vars_file) -> None:
if env_vars_file:
return rm_rf(env_vars_file)