def download_and_extract_with_retry(archive_url, tmp_archive, target_dir): try: download_and_extract(archive_url, target_dir, tmp_archive=tmp_archive) except Exception as e: # try deleting and re-downloading the zip file LOG.info("Unable to extract file, re-downloading ZIP archive %s: %s", tmp_archive, e) rm_rf(tmp_archive) download_and_extract(archive_url, target_dir, tmp_archive=tmp_archive)
def create_dynamodb_server(port=None, db_path: Optional[str] = None, clean_db_path: bool = False) -> DynamodbServer: """ Creates a dynamodb server from the LocalStack configuration. """ port = port or get_free_tcp_port() server = DynamodbServer(port) db_path = f"{config.dirs.data}/dynamodb" if not db_path and config.dirs.data else db_path if db_path: if clean_db_path: rm_rf(db_path) mkdir(db_path) absolute_path = os.path.abspath(db_path) server.db_path = absolute_path server.heap_size = config.DYNAMODB_HEAP_SIZE server.share_db = is_env_true("DYNAMODB_SHARE_DB") server.optimize_db_before_startup = is_env_true( "DYNAMODB_OPTIMIZE_DB_BEFORE_STARTUP") server.delay_transient_statuses = is_env_true( "DYNAMODB_DELAY_TRANSIENT_STATUSES") server.cors = os.getenv("DYNAMODB_CORS", None) return server
def create_lambda_archive( script: str, get_content: bool = False, libs: List[str] = None, runtime: str = None, file_name: str = None, exclude_func: Callable[[str], bool] = None, ): """Utility method to create a Lambda function archive""" if libs is None: libs = [] runtime = runtime or LAMBDA_DEFAULT_RUNTIME with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir: file_name = file_name or get_handler_file_from_name( LAMBDA_DEFAULT_HANDLER, runtime=runtime) script_file = os.path.join(tmp_dir, file_name) if os.path.sep in script_file: mkdir(os.path.dirname(script_file)) # create __init__.py files along the path to allow Python imports path = file_name.split(os.path.sep) for i in range(1, len(path)): save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "") save_file(script_file, script) chmod_r(script_file, 0o777) # copy libs for lib in libs: paths = [lib, "%s.py" % lib] try: module = importlib.import_module(lib) paths.append(module.__file__) except Exception: pass target_dir = tmp_dir root_folder = os.path.join(LOCALSTACK_VENV_FOLDER, "lib/python*/site-packages") if lib == "localstack": paths = ["localstack/*.py", "localstack/utils"] root_folder = LOCALSTACK_ROOT_FOLDER target_dir = os.path.join(tmp_dir, lib) mkdir(target_dir) for path in paths: file_to_copy = path if path.startswith("/") else os.path.join( root_folder, path) for file_path in glob.glob(file_to_copy): name = os.path.join(target_dir, file_path.split(os.path.sep)[-1]) if os.path.isdir(file_path): copy_dir(file_path, name) else: shutil.copyfile(file_path, name) if exclude_func: for dirpath, folders, files in os.walk(tmp_dir): for name in list(folders) + list(files): full_name = os.path.join(dirpath, name) relative = os.path.relpath(full_name, start=tmp_dir) if exclude_func(relative): rm_rf(full_name) # create zip file result = create_zip_file(tmp_dir, get_content=get_content) return result
def install_stepfunctions_local(): if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR): # pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file if not DOCKER_CLIENT.has_docker(): # TODO: works only when a docker socket is available -> add a fallback if running without Docker? LOG.warning( "Docker not available - skipping installation of StepFunctions dependency" ) return log_install_msg("Step Functions") mkdir(INSTALL_DIR_STEPFUNCTIONS) DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL) docker_name = "tmp-ls-sfn" DOCKER_CLIENT.run_container( IMAGE_NAME_SFN_LOCAL, remove=True, entrypoint="", name=docker_name, detach=True, command=["sleep", "15"], ) time.sleep(5) DOCKER_CLIENT.copy_from_container( docker_name, local_path=dirs.static_libs, container_path="/home/stepfunctionslocal/") path = Path(f"{dirs.static_libs}/stepfunctionslocal/") for file in path.glob("*.jar"): file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name) rm_rf(str(path)) classes = [ SFN_PATCH_CLASS1, SFN_PATCH_CLASS2, SFN_PATCH_CLASS_REGION, SFN_PATCH_CLASS_STARTER, SFN_PATCH_CLASS_ASYNC2SERVICEAPI, SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED, SFN_PATCH_FILE_METAINF, ] for patch_class in classes: patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}" add_file_to_jar(patch_class, patch_url, target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR) # special case for Manifest file - extract first, replace content, then update in JAR file manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF", "MANIFEST.MF") if not os.path.exists(manifest_file): content = run([ "unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF" ]) content = re.sub("Main-Class: .+", "Main-Class: cloud.localstack.StepFunctionsStarter", content) classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS]) content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ", content) save_file(manifest_file, content) run( ["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"], cwd=INSTALL_DIR_STEPFUNCTIONS, ) # download additional jar libs for jar_url in JAR_URLS: target = os.path.join(INSTALL_DIR_STEPFUNCTIONS, os.path.basename(jar_url)) if not file_exists_not_empty(target): download(jar_url, target) # download aws-sdk lambda handler target = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "localstack-internal-awssdk", "awssdk.zip") if not file_exists_not_empty(target): download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def install_elasticsearch(version=None): # locally import to avoid having a dependency on ASF when starting the CLI from localstack.aws.api.opensearch import EngineType from localstack.services.opensearch import versions if not version: version = ELASTICSEARCH_DEFAULT_VERSION version = get_elasticsearch_install_version(version) install_dir = get_elasticsearch_install_dir(version) installed_executable = os.path.join(install_dir, "bin", "elasticsearch") if not os.path.exists(installed_executable): log_install_msg(f"Elasticsearch ({version})") es_url = versions.get_download_url(version, EngineType.Elasticsearch) install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join(config.dirs.tmp, f"localstack.{os.path.basename(es_url)}") download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob( os.path.join(install_dir_parent, "elasticsearch*")) if not elasticsearch_dir: raise Exception( f"Unable to find Elasticsearch folder in {install_dir_parent}") shutil.move(elasticsearch_dir[0], install_dir) for dir_name in ("data", "logs", "modules", "plugins", "config/scripts"): dir_path = os.path.join(install_dir, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # install default plugins for plugin in ELASTICSEARCH_PLUGIN_LIST: plugin_binary = os.path.join(install_dir, "bin", "elasticsearch-plugin") plugin_dir = os.path.join(install_dir, "plugins", plugin) if not os.path.exists(plugin_dir): LOG.info("Installing Elasticsearch plugin %s", plugin) def try_install(): output = run([plugin_binary, "install", "-b", plugin]) LOG.debug("Plugin installation output: %s", output) # We're occasionally seeing javax.net.ssl.SSLHandshakeException -> add download retries download_attempts = 3 try: retry(try_install, retries=download_attempts - 1, sleep=2) except Exception: LOG.warning( "Unable to download Elasticsearch plugin '%s' after %s attempts", plugin, download_attempts, ) if not os.environ.get("IGNORE_ES_DOWNLOAD_ERRORS"): raise # delete some plugins to free up space for plugin in ELASTICSEARCH_DELETE_MODULES: module_dir = os.path.join(install_dir, "modules", plugin) rm_rf(module_dir) # disable x-pack-ml plugin (not working on Alpine) xpack_dir = os.path.join(install_dir, "modules", "x-pack-ml", "platform") rm_rf(xpack_dir) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(install_dir, "config", "jvm.options") if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)", r"# \1", jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def listen_to_kinesis( stream_name, listener_func=None, processor_script=None, events_file=None, endpoint_url=None, log_file=None, configs=None, env=None, ddb_lease_table_suffix=None, env_vars=None, kcl_log_level=DEFAULT_KCL_LOG_LEVEL, log_subscribers=None, wait_until_started=False, fh_d_stream=None, region_name=None, ): """ High-level function that allows to subscribe to a Kinesis stream and receive events in a listener function. A KCL client process is automatically started in the background. """ if configs is None: configs = {} if env_vars is None: env_vars = {} if log_subscribers is None: log_subscribers = [] env = aws_stack.get_environment(env) if not events_file: events_file = EVENTS_FILE_PATTERN.replace("*", short_uid()) TMP_FILES.append(events_file) if not processor_script: processor_script = generate_processor_script(events_file, log_file=log_file) rm_rf(events_file) # start event reader thread (this process) ready_mutex = threading.Semaphore(0) thread = EventFileReaderThread(events_file, listener_func, ready_mutex=ready_mutex, fh_d_stream=fh_d_stream) thread.start() # Wait until the event reader thread is ready (to avoid 'Connection refused' error on the UNIX socket) ready_mutex.acquire() # start KCL client (background process) if processor_script[-4:] == ".pyc": processor_script = processor_script[0:-1] # add log listener that notifies when KCL is started if wait_until_started: listener = KclStartedLogListener() log_subscribers.append(listener) process = start_kcl_client_process( stream_name, processor_script, endpoint_url=endpoint_url, log_file=log_file, configs=configs, env=env, ddb_lease_table_suffix=ddb_lease_table_suffix, env_vars=env_vars, kcl_log_level=kcl_log_level, log_subscribers=log_subscribers, region_name=region_name, ) if wait_until_started: # Wait at most 90 seconds for initialization. Note that creating the DDB table can take quite a bit try: listener.sync_init.get(block=True, timeout=90) except Exception: raise Exception("Timeout when waiting for KCL initialization.") # wait at most 30 seconds for shard lease notification try: listener.sync_take_shard.get(block=True, timeout=30) except Exception: # this merely means that there is no shard available to take. Do nothing. pass return process
def rm_env_vars_file(env_vars_file) -> None: if env_vars_file: return rm_rf(env_vars_file)