def create_env_vars_file_flag( cls, env_vars: Dict) -> Tuple[List[str], Optional[str]]: if not env_vars: return [], None result = [] env_vars = dict(env_vars) env_file = None if len(str(env_vars)) > cls.MAX_ENV_ARGS_LENGTH: # default ARG_MAX=131072 in Docker - let's create an env var file if the string becomes too long... env_file = cls.mountable_tmp_file() env_content = "" for name, value in dict(env_vars).items(): if len(value) > cls.MAX_ENV_ARGS_LENGTH: # each line in the env file has a max size as well (error "bufio.Scanner: token too long") continue env_vars.pop(name) value = value.replace("\n", "\\") env_content += f"{cls.format_env_vars(name, value)}\n" save_file(env_file, env_content) result += ["--env-file", env_file] env_vars_res = [ item for k, v in env_vars.items() for item in ["-e", cls.format_env_vars(k, v)] ] result += env_vars_res return result, env_file
def download_and_extract(archive_url, target_dir, retries=0, sleep=3, tmp_archive=None): mkdir(target_dir) _, ext = os.path.splitext(tmp_archive or archive_url) tmp_archive = tmp_archive or new_tmp_file() if not os.path.exists(tmp_archive) or os.path.getsize(tmp_archive) <= 0: # create temporary placeholder file, to avoid duplicate parallel downloads save_file(tmp_archive, "") for i in range(retries + 1): try: download(archive_url, tmp_archive) break except Exception: time.sleep(sleep) if ext == ".zip": unzip(tmp_archive, target_dir) elif ext in [".bz2", ".gz", ".tgz"]: untar(tmp_archive, target_dir) else: raise Exception(f"Unsupported archive format: {ext}")
def create_zip_file(file_path, zip_file=None, get_content=False, content_root=None, mode="w"): """ Creates a zipfile to the designated file_path. By default, a new zip file is created but the mode parameter can be used to append to an existing zip file """ base_dir = file_path if not os.path.isdir(file_path): base_dir = tempfile.mkdtemp(prefix=ARCHIVE_DIR_PREFIX) shutil.copy(file_path, base_dir) TMP_FILES.append(base_dir) tmp_dir = tempfile.mkdtemp(prefix=ARCHIVE_DIR_PREFIX) full_zip_file = zip_file if not full_zip_file: zip_file_name = "archive.zip" full_zip_file = os.path.join(tmp_dir, zip_file_name) # special case where target folder is empty -> create empty zip file if is_empty_dir(base_dir): # see https://stackoverflow.com/questions/25195495/how-to-create-an-empty-zip-file#25195628 content = ( b"PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" ) if get_content: return content save_file(full_zip_file, content) return full_zip_file # create zip file if is_debian(): # todo: extend CLI with the new parameters create_zip_file_cli(source_path=file_path, base_dir=base_dir, zip_file=full_zip_file) else: create_zip_file_python( source_path=file_path, base_dir=base_dir, zip_file=full_zip_file, content_root=content_root, mode=mode, ) if not get_content: TMP_FILES.append(tmp_dir) return full_zip_file with open(full_zip_file, "rb") as file_obj: zip_file_content = file_obj.read() rm_dir(tmp_dir) return zip_file_content
def install_opensearch(version=None): # locally import to avoid having a dependency on ASF when starting the CLI from localstack.aws.api.opensearch import EngineType from localstack.services.opensearch import versions if not version: version = OPENSEARCH_DEFAULT_VERSION version = get_opensearch_install_version(version) install_dir = get_opensearch_install_dir(version) installed_executable = os.path.join(install_dir, "bin", "opensearch") if not os.path.exists(installed_executable): with OS_INSTALL_LOCKS.setdefault(version, threading.Lock()): if not os.path.exists(installed_executable): log_install_msg(f"OpenSearch ({version})") opensearch_url = versions.get_download_url( version, EngineType.OpenSearch) install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join( config.dirs.tmp, f"localstack.{os.path.basename(opensearch_url)}") download_and_extract_with_retry(opensearch_url, tmp_archive, install_dir_parent) opensearch_dir = glob.glob( os.path.join(install_dir_parent, "opensearch*")) if not opensearch_dir: raise Exception( f"Unable to find OpenSearch folder in {install_dir_parent}" ) shutil.move(opensearch_dir[0], install_dir) for dir_name in ("data", "logs", "modules", "plugins", "config/scripts"): dir_path = os.path.join(install_dir, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(install_dir, "config", "jvm.options") if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)", r"# \1", jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def _save_cert_keys(client_cert_key: Tuple[str, str]) -> Tuple[str, str]: """ Save the given cert / key into files and returns their filename :param client_cert_key: tuple with (client_cert, client_key) :return: tuple of paths to files containing (client_cert, client_key) """ cert_file = client_cert_key[0] if not os.path.exists(cert_file): cert_file = new_tmp_file() save_file(cert_file, client_cert_key[0]) key_file = client_cert_key[1] if not os.path.exists(key_file): key_file = new_tmp_file() save_file(key_file, client_cert_key[1]) return cert_file, key_file
def _do_start_ssl_proxy_with_client_auth(port: int, target: PortOrUrl, client_cert_key: Tuple[str, str]): # prepare cert files (TODO: check whether/how we can pass cert strings to requests.request(..) directly) cert_file = client_cert_key[0] if not os.path.exists(cert_file): cert_file = new_tmp_file() save_file(cert_file, client_cert_key[0]) key_file = client_cert_key[1] if not os.path.exists(key_file): key_file = new_tmp_file() save_file(key_file, client_cert_key[1]) cert_params = (cert_file, key_file) # start proxy requests_kwargs = {"cert": cert_params} result = _do_start_ssl_proxy_with_listener(port, target, requests_kwargs=requests_kwargs) return result
def install_dynamodb_local(): if not os.path.exists(INSTALL_PATH_DDB_JAR): log_install_msg("DynamoDB") # download and extract archive tmp_archive = os.path.join(tempfile.gettempdir(), "localstack.ddb.zip") download_and_extract_with_retry(DYNAMODB_JAR_URL, tmp_archive, INSTALL_DIR_DDB) # fix logging configuration for DynamoDBLocal log4j2_config = """<Configuration status="WARN"> <Appenders> <Console name="Console" target="SYSTEM_OUT"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> </Console> </Appenders> <Loggers> <Root level="WARN"><AppenderRef ref="Console"/></Root> </Loggers> </Configuration>""" log4j2_file = os.path.join(INSTALL_DIR_DDB, "log4j2.xml") save_file(log4j2_file, log4j2_config) run_safe(lambda: run(["zip", "-u", "DynamoDBLocal.jar", "log4j2.xml"], cwd=INSTALL_DIR_DDB)) # download agent JAR if not os.path.exists(DDB_AGENT_JAR_PATH): download(DDB_AGENT_JAR_URL, DDB_AGENT_JAR_PATH) if not os.path.exists(JAVASSIST_JAR_PATH): download(JAVASSIST_JAR_URL, JAVASSIST_JAR_PATH) # ensure that javassist.jar is in the manifest classpath run(["unzip", "-o", "DynamoDBLocal.jar", "META-INF/MANIFEST.MF"], cwd=INSTALL_DIR_DDB) manifest_file = os.path.join(INSTALL_DIR_DDB, "META-INF", "MANIFEST.MF") manifest = load_file(manifest_file) if "javassist.jar" not in manifest: manifest = manifest.replace("Class-Path:", "Class-Path: javassist.jar", 1) save_file(manifest_file, manifest) run(["zip", "-u", "DynamoDBLocal.jar", "META-INF/MANIFEST.MF"], cwd=INSTALL_DIR_DDB)
def create_config_file( config_file, executableName, streamName, applicationName, credentialsProvider=None, region_name=None, **kwargs, ): if not credentialsProvider: credentialsProvider = "DefaultAWSCredentialsProviderChain" region_name = region_name or aws_stack.get_region() content = """ executableName = %s streamName = %s applicationName = %s AWSCredentialsProvider = %s processingLanguage = python/2.7 parentShardPollIntervalMillis = 2000 regionName = %s """ % ( executableName, streamName, applicationName, credentialsProvider, region_name, ) # optional properties for key, value in kwargs.items(): content += """ %s = %s""" % ( key, value, ) content = content.replace(" ", "") save_file(config_file, content)
def create_lambda_archive( script: str, get_content: bool = False, libs: List[str] = None, runtime: str = None, file_name: str = None, exclude_func: Callable[[str], bool] = None, ): """Utility method to create a Lambda function archive""" if libs is None: libs = [] runtime = runtime or LAMBDA_DEFAULT_RUNTIME with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir: file_name = file_name or get_handler_file_from_name( LAMBDA_DEFAULT_HANDLER, runtime=runtime) script_file = os.path.join(tmp_dir, file_name) if os.path.sep in script_file: mkdir(os.path.dirname(script_file)) # create __init__.py files along the path to allow Python imports path = file_name.split(os.path.sep) for i in range(1, len(path)): save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "") save_file(script_file, script) chmod_r(script_file, 0o777) # copy libs for lib in libs: paths = [lib, "%s.py" % lib] try: module = importlib.import_module(lib) paths.append(module.__file__) except Exception: pass target_dir = tmp_dir root_folder = os.path.join(LOCALSTACK_VENV_FOLDER, "lib/python*/site-packages") if lib == "localstack": paths = ["localstack/*.py", "localstack/utils"] root_folder = LOCALSTACK_ROOT_FOLDER target_dir = os.path.join(tmp_dir, lib) mkdir(target_dir) for path in paths: file_to_copy = path if path.startswith("/") else os.path.join( root_folder, path) for file_path in glob.glob(file_to_copy): name = os.path.join(target_dir, file_path.split(os.path.sep)[-1]) if os.path.isdir(file_path): copy_dir(file_path, name) else: shutil.copyfile(file_path, name) if exclude_func: for dirpath, folders, files in os.walk(tmp_dir): for name in list(folders) + list(files): full_name = os.path.join(dirpath, name) relative = os.path.relpath(full_name, start=tmp_dir) if exclude_func(relative): rm_rf(full_name) # create zip file result = create_zip_file(tmp_dir, get_content=get_content) return result
def install_stepfunctions_local(): if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR): # pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file if not DOCKER_CLIENT.has_docker(): # TODO: works only when a docker socket is available -> add a fallback if running without Docker? LOG.warning( "Docker not available - skipping installation of StepFunctions dependency" ) return log_install_msg("Step Functions") mkdir(INSTALL_DIR_STEPFUNCTIONS) DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL) docker_name = "tmp-ls-sfn" DOCKER_CLIENT.run_container( IMAGE_NAME_SFN_LOCAL, remove=True, entrypoint="", name=docker_name, detach=True, command=["sleep", "15"], ) time.sleep(5) DOCKER_CLIENT.copy_from_container( docker_name, local_path=dirs.static_libs, container_path="/home/stepfunctionslocal/") path = Path(f"{dirs.static_libs}/stepfunctionslocal/") for file in path.glob("*.jar"): file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name) rm_rf(str(path)) classes = [ SFN_PATCH_CLASS1, SFN_PATCH_CLASS2, SFN_PATCH_CLASS_REGION, SFN_PATCH_CLASS_STARTER, SFN_PATCH_CLASS_ASYNC2SERVICEAPI, SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED, SFN_PATCH_FILE_METAINF, ] for patch_class in classes: patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}" add_file_to_jar(patch_class, patch_url, target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR) # special case for Manifest file - extract first, replace content, then update in JAR file manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF", "MANIFEST.MF") if not os.path.exists(manifest_file): content = run([ "unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF" ]) content = re.sub("Main-Class: .+", "Main-Class: cloud.localstack.StepFunctionsStarter", content) classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS]) content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ", content) save_file(manifest_file, content) run( ["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"], cwd=INSTALL_DIR_STEPFUNCTIONS, ) # download additional jar libs for jar_url in JAR_URLS: target = os.path.join(INSTALL_DIR_STEPFUNCTIONS, os.path.basename(jar_url)) if not file_exists_not_empty(target): download(jar_url, target) # download aws-sdk lambda handler target = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "localstack-internal-awssdk", "awssdk.zip") if not file_exists_not_empty(target): download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def install_elasticsearch(version=None): # locally import to avoid having a dependency on ASF when starting the CLI from localstack.aws.api.opensearch import EngineType from localstack.services.opensearch import versions if not version: version = ELASTICSEARCH_DEFAULT_VERSION version = get_elasticsearch_install_version(version) install_dir = get_elasticsearch_install_dir(version) installed_executable = os.path.join(install_dir, "bin", "elasticsearch") if not os.path.exists(installed_executable): log_install_msg(f"Elasticsearch ({version})") es_url = versions.get_download_url(version, EngineType.Elasticsearch) install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join(config.dirs.tmp, f"localstack.{os.path.basename(es_url)}") download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob( os.path.join(install_dir_parent, "elasticsearch*")) if not elasticsearch_dir: raise Exception( f"Unable to find Elasticsearch folder in {install_dir_parent}") shutil.move(elasticsearch_dir[0], install_dir) for dir_name in ("data", "logs", "modules", "plugins", "config/scripts"): dir_path = os.path.join(install_dir, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # install default plugins for plugin in ELASTICSEARCH_PLUGIN_LIST: plugin_binary = os.path.join(install_dir, "bin", "elasticsearch-plugin") plugin_dir = os.path.join(install_dir, "plugins", plugin) if not os.path.exists(plugin_dir): LOG.info("Installing Elasticsearch plugin %s", plugin) def try_install(): output = run([plugin_binary, "install", "-b", plugin]) LOG.debug("Plugin installation output: %s", output) # We're occasionally seeing javax.net.ssl.SSLHandshakeException -> add download retries download_attempts = 3 try: retry(try_install, retries=download_attempts - 1, sleep=2) except Exception: LOG.warning( "Unable to download Elasticsearch plugin '%s' after %s attempts", plugin, download_attempts, ) if not os.environ.get("IGNORE_ES_DOWNLOAD_ERRORS"): raise # delete some plugins to free up space for plugin in ELASTICSEARCH_DELETE_MODULES: module_dir = os.path.join(install_dir, "modules", plugin) rm_rf(module_dir) # disable x-pack-ml plugin (not working on Alpine) xpack_dir = os.path.join(install_dir, "modules", "x-pack-ml", "platform") rm_rf(xpack_dir) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(install_dir, "config", "jvm.options") if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)", r"# \1", jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def generate_processor_script(events_file, log_file=None): script_file = os.path.join(tempfile.gettempdir(), "kclipy.%s.processor.py" % short_uid()) if log_file: log_file = "'%s'" % log_file else: log_file = "None" content = """#!/usr/bin/env python import os, sys, glob, json, socket, time, logging, subprocess, tempfile logging.basicConfig(level=logging.INFO) for path in glob.glob('%s/lib/python*/site-packages'): sys.path.insert(0, path) sys.path.insert(0, '%s') from localstack.config import DEFAULT_ENCODING from localstack.utils.kinesis import kinesis_connector from localstack.utils.time import timestamp events_file = '%s' log_file = %s error_log = os.path.join(tempfile.gettempdir(), 'kclipy.error.log') if __name__ == '__main__': sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) num_tries = 3 sleep_time = 2 error = None for i in range(0, num_tries): try: sock.connect(events_file) error = None break except Exception as e: error = e if i < num_tries: msg = '%%s: Unable to connect to UNIX socket. Retrying.' %% timestamp() subprocess.check_output('echo "%%s" >> %%s' %% (msg, error_log), shell=True) time.sleep(sleep_time) if error: print("WARN: Unable to connect to UNIX socket after retrying: %%s" %% error) raise error def receive_msg(records, checkpointer, shard_id): try: # records is a list of amazon_kclpy.messages.Record objects -> convert to JSON records_dicts = [j._json_dict for j in records] message_to_send = {'shard_id': shard_id, 'records': records_dicts} string_to_send = '%%s\\n' %% json.dumps(message_to_send) bytes_to_send = string_to_send.encode(DEFAULT_ENCODING) sock.send(bytes_to_send) except Exception as e: msg = "WARN: Unable to forward event: %%s" %% e print(msg) subprocess.check_output('echo "%%s" >> %%s' %% (msg, error_log), shell=True) kinesis_connector.KinesisProcessor.run_processor(log_file=log_file, processor_func=receive_msg) """ % ( LOCALSTACK_VENV_FOLDER, LOCALSTACK_ROOT_FOLDER, events_file, log_file, ) save_file(script_file, content) chmod_r(script_file, 0o755) TMP_FILES.append(script_file) return script_file
def log(self, s): s = "%s\n" % s if self.log_file: save_file(self.log_file, s, append=True)