def install_dynamodb_local(): if not os.path.exists(INSTALL_DIR_DDB): LOGGER.info('Downloading and installing local DynamoDB server. This may take some time.') mkdir(INSTALL_DIR_DDB) # download and extract archive download_and_extract_with_retry(DYNAMODB_JAR_URL, TMP_ARCHIVE_DDB, INSTALL_DIR_DDB) # fix for Alpine, otherwise DynamoDBLocal fails with: # DynamoDBLocal_lib/libsqlite4java-linux-amd64.so: __memcpy_chk: symbol not found if is_alpine(): ddb_libs_dir = '%s/DynamoDBLocal_lib' % INSTALL_DIR_DDB patched_marker = '%s/alpine_fix_applied' % ddb_libs_dir if not os.path.exists(patched_marker): patched_lib = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' + 'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/libsqlite4java-linux-amd64.so') patched_jar = ('https://rawgit.com/bhuisgen/docker-alpine/master/alpine-dynamodb/' + 'rootfs/usr/local/dynamodb/DynamoDBLocal_lib/sqlite4java.jar') run("curl -L -o %s/libsqlite4java-linux-amd64.so '%s'" % (ddb_libs_dir, patched_lib)) run("curl -L -o %s/sqlite4java.jar '%s'" % (ddb_libs_dir, patched_jar)) save_file(patched_marker, '') # fix logging configuration for DynamoDBLocal log4j2_config = """<Configuration status="WARN"> <Appenders> <Console name="Console" target="SYSTEM_OUT"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> </Console> </Appenders> <Loggers> <Root level="WARN"><AppenderRef ref="Console"/></Root> </Loggers> </Configuration>""" log4j2_file = os.path.join(INSTALL_DIR_DDB, 'log4j2.xml') save_file(log4j2_file, log4j2_config) run('cd "%s" && zip -u DynamoDBLocal.jar log4j2.xml || true' % INSTALL_DIR_DDB)
def setUpClass(cls): cls.lambda_client = aws_stack.connect_to_service('lambda') # deploy lambda - Java if not os.path.exists(TEST_LAMBDA_JAVA): mkdir(os.path.dirname(TEST_LAMBDA_JAVA)) download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA) # Lambda supports single JAR deployments without the zip, # so we upload the JAR directly. cls.test_java_jar = load_file(TEST_LAMBDA_JAVA, mode='rb') cls.test_java_zip = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA, zip_file=cls.test_java_jar, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler') # deploy lambda - Java with stream handler testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=cls.test_java_jar, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler') # deploy lambda - Java with serializable input object testutil.create_lambda_function( func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, zip_file=cls.test_java_zip, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.SerializedInputLambdaHandler')
def start_elasticsearch(port=None, delete_data=True, asynchronous=False, update_listener=None): port = port or config.PORT_ELASTICSEARCH # delete Elasticsearch data that may be cached locally from a previous test run delete_all_elasticsearch_data() install.install_elasticsearch() backend_port = DEFAULT_PORT_ELASTICSEARCH_BACKEND es_data_dir = '%s/infra/elasticsearch/data' % (ROOT_PATH) es_tmp_dir = '%s/infra/elasticsearch/tmp' % (ROOT_PATH) if config.DATA_DIR: es_data_dir = '%s/elasticsearch' % config.DATA_DIR # Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments, # hence we use the default bind address 127.0.0.0 and put a proxy in front of it cmd = (('%s/infra/elasticsearch/bin/elasticsearch ' + '-E http.port=%s -E http.publish_port=%s -E http.compression=false -E path.data=%s') % (ROOT_PATH, backend_port, backend_port, es_data_dir)) env_vars = { 'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'), 'ES_TMPDIR': es_tmp_dir } print('Starting local Elasticsearch (%s port %s)...' % (get_service_protocol(), port)) if delete_data: run('rm -rf %s' % es_data_dir) # fix permissions chmod_r('%s/infra/elasticsearch' % ROOT_PATH, 0o777) mkdir(es_data_dir) chmod_r(es_data_dir, 0o777) # start proxy and ES process start_proxy_for_service('elasticsearch', port, backend_port, update_listener, quiet=True, params={'protocol_version': 'HTTP/1.0'}) if is_root(): cmd = "su -c '%s' localstack" % cmd thread = do_run(cmd, asynchronous, env_vars=env_vars) return thread
def get_lambda_code_param(params, **kwargs): code = params.get("Code", {}) zip_file = code.get("ZipFile") if zip_file and not is_base64(zip_file): tmp_dir = new_tmp_dir() handler_file = get_handler_file_from_name( params["Handler"], runtime=params["Runtime"]) tmp_file = os.path.join(tmp_dir, handler_file) save_file(tmp_file, zip_file) # add 'cfn-response' module to archive - see: # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-lambda-function-code-cfnresponsemodule.html cfn_response_tmp_file = get_cfn_response_mod_file() cfn_response_mod_dir = os.path.join(tmp_dir, "node_modules", "cfn-response") mkdir(cfn_response_mod_dir) cp_r( cfn_response_tmp_file, os.path.join(cfn_response_mod_dir, "index.js"), ) # create zip file zip_file = create_zip_file(tmp_dir, get_content=True) code["ZipFile"] = zip_file rm_rf(tmp_dir) return code
def download_and_extract(archive_url, target_dir, retries=0, sleep=3, tmp_archive=None): mkdir(target_dir) tmp_archive = tmp_archive or new_tmp_file() if not os.path.exists(tmp_archive): # create temporary placeholder file, to avoid duplicate parallel downloads save_file(tmp_archive, '') for i in range(retries + 1): try: download(archive_url, tmp_archive) break except Exception: time.sleep(sleep) _, ext = os.path.splitext(tmp_archive) if ext == '.zip': unzip(tmp_archive, target_dir) elif ext == '.gz' or ext == '.bz2': untar(tmp_archive, target_dir) else: raise Exception('Unsupported archive format: %s' % ext)
def install_stepfunctions_local(): if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR): # pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file log_install_msg('Step Functions') mkdir(INSTALL_DIR_STEPFUNCTIONS) run('{dc} pull {img}'.format(dc=config.DOCKER_CMD, img=IMAGE_NAME_SFN_LOCAL)) docker_name = 'tmp-ls-sfn' run(('{dc} run --name={dn} --entrypoint= -d --rm {img} sleep 15' ).format(dc=config.DOCKER_CMD, dn=docker_name, img=IMAGE_NAME_SFN_LOCAL)) time.sleep(5) run('{dc} cp {dn}:/home/stepfunctionslocal/ {tgt}'.format( dc=config.DOCKER_CMD, dn=docker_name, tgt=INSTALL_DIR_INFRA)) run('mv %s/stepfunctionslocal/*.jar %s' % (INSTALL_DIR_INFRA, INSTALL_DIR_STEPFUNCTIONS)) rm_rf('%s/stepfunctionslocal' % INSTALL_DIR_INFRA) # apply patches patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, SFN_PATCH_CLASS) if not os.path.exists(patch_class_file): download(SFN_PATCH_CLASS_URL, patch_class_file) cmd = 'cd "%s"; zip %s %s' % (INSTALL_DIR_STEPFUNCTIONS, INSTALL_PATH_STEPFUNCTIONS_JAR, SFN_PATCH_CLASS) run(cmd)
def create_lambda_archive( script: str, get_content: bool = False, libs: List[str] = None, runtime: str = None, file_name: str = None, exclude_func: Callable[[str], bool] = None, ): """Utility method to create a Lambda function archive""" if libs is None: libs = [] runtime = runtime or LAMBDA_DEFAULT_RUNTIME with tempfile.TemporaryDirectory(prefix=ARCHIVE_DIR_PREFIX) as tmp_dir: file_name = file_name or get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER, runtime=runtime) script_file = os.path.join(tmp_dir, file_name) if os.path.sep in script_file: mkdir(os.path.dirname(script_file)) # create __init__.py files along the path to allow Python imports path = file_name.split(os.path.sep) for i in range(1, len(path)): save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "") save_file(script_file, script) chmod_r(script_file, 0o777) # copy libs for lib in libs: paths = [lib, "%s.py" % lib] try: module = importlib.import_module(lib) paths.append(module.__file__) except Exception: pass target_dir = tmp_dir root_folder = os.path.join(LOCALSTACK_VENV_FOLDER, "lib/python*/site-packages") if lib == "localstack": paths = ["localstack/*.py", "localstack/utils"] root_folder = LOCALSTACK_ROOT_FOLDER target_dir = os.path.join(tmp_dir, lib) mkdir(target_dir) for path in paths: file_to_copy = path if path.startswith("/") else os.path.join(root_folder, path) for file_path in glob.glob(file_to_copy): name = os.path.join(target_dir, file_path.split(os.path.sep)[-1]) if os.path.isdir(file_path): copy_dir(file_path, name) else: shutil.copyfile(file_path, name) if exclude_func: for dirpath, folders, files in os.walk(tmp_dir): for name in list(folders) + list(files): full_name = os.path.join(dirpath, name) relative = os.path.relpath(full_name, start=tmp_dir) if exclude_func(relative): rm_rf(full_name) # create zip file result = create_zip_file(tmp_dir, get_content=get_content) return result
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None): kinesis_mock_bin = install.install_kinesis_mock() backend_port = get_free_tcp_port() global PORT_KINESIS_BACKEND PORT_KINESIS_BACKEND = backend_port kinesis_data_dir_param = "" if config.DATA_DIR: kinesis_data_dir = "%s/kinesis" % config.DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = "SHOULD_PERSIST_DATA=true PERSIST_PATH=%s" % kinesis_data_dir if not config.LS_LOG: log_level = "INFO" elif config.LS_LOG == "warning": log_level = "WARN" else: log_level = config.LS_LOG.upper() log_level_param = "LOG_LEVEL=%s" % log_level latency = config.KINESIS_LATENCY + "ms" latency_param = ( "CREATE_STREAM_DURATION={l} DELETE_STREAM_DURATION={l} REGISTER_STREAM_CONSUMER_DURATION={l} " "START_STREAM_ENCRYPTION_DURATION={l} STOP_STREAM_ENCRYPTION_DURATION={l} " "DEREGISTER_STREAM_CONSUMER_DURATION={l} MERGE_SHARDS_DURATION={l} SPLIT_SHARD_DURATION={l} " "UPDATE_SHARD_COUNT_DURATION={l}").format(l=latency) if config.KINESIS_INITIALIZE_STREAMS != "": initialize_streams_param = "INITIALIZE_STREAMS=%s" % config.KINESIS_INITIALIZE_STREAMS else: initialize_streams_param = "" if kinesis_mock_bin.endswith(".jar"): cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s" % ( backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, initialize_streams_param, kinesis_mock_bin, ) else: chmod_r(kinesis_mock_bin, 0o777) cmd = "KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1" % ( backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, initialize_streams_param, kinesis_mock_bin, ) return _run_proxy_and_command( cmd=cmd, port=port, backend_port=backend_port, update_listener=update_listener, asynchronous=asynchronous, )
def install_elasticmq(): if not os.path.exists(INSTALL_DIR_ELASTICMQ): LOGGER.info('Downloading and installing local ElasticMQ server. This may take some time.') mkdir(INSTALL_DIR_ELASTICMQ) # download archive if not os.path.exists(TMP_ARCHIVE_ELASTICMQ): download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ) shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)
def install_local_kms(): binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace('<arch>', get_arch()) if not os.path.exists(binary_path): log_install_msg('KMS') mkdir(INSTALL_DIR_KMS) kms_url = KMS_URL_PATTERN.replace('<arch>', get_arch()) download(kms_url, binary_path) chmod_r(binary_path, 0o777)
def test_get_java_lib_folder_classpath_no_directories(self): base_dir = new_tmp_dir() jar_file = os.path.join(base_dir, 'foo.jar') save_file(jar_file, '') lib_file = os.path.join(base_dir, 'lib', 'lib.jar') mkdir(os.path.dirname(lib_file)) save_file(lib_file, '') self.assertEquals('.:foo.jar:lib/lib.jar', lambda_executors.Util.get_java_classpath(jar_file))
def start_elasticsearch(port=None, version=None, delete_data=True, asynchronous=False, update_listener=None): if STATE.get('_thread_'): return STATE['_thread_'] port = port or config.PORT_ELASTICSEARCH # delete Elasticsearch data that may be cached locally from a previous test run delete_all_elasticsearch_data(version) install.install_elasticsearch(version) backend_port = get_free_tcp_port() base_dir = install.get_elasticsearch_install_dir(version) es_data_dir = os.path.join(base_dir, 'data') es_tmp_dir = os.path.join(base_dir, 'tmp') es_mods_dir = os.path.join(base_dir, 'modules') if config.DATA_DIR: delete_data = False es_data_dir = '%s/elasticsearch' % config.DATA_DIR # Elasticsearch 5.x cannot be bound to 0.0.0.0 in some Docker environments, # hence we use the default bind address 127.0.0.0 and put a proxy in front of it backup_dir = os.path.join(config.TMP_FOLDER, 'es_backup') cmd = ( ('%s/bin/elasticsearch ' + '-E http.port=%s -E http.publish_port=%s -E http.compression=false ' + '-E path.data=%s -E path.repo=%s') % (base_dir, backend_port, backend_port, es_data_dir, backup_dir)) if os.path.exists(os.path.join(es_mods_dir, 'x-pack-ml')): cmd += ' -E xpack.ml.enabled=false' env_vars = { 'ES_JAVA_OPTS': os.environ.get('ES_JAVA_OPTS', '-Xms200m -Xmx600m'), 'ES_TMPDIR': es_tmp_dir } LOG.debug('Starting local Elasticsearch (%s port %s)' % (get_service_protocol(), port)) if delete_data: rm_rf(es_data_dir) # fix permissions chmod_r(base_dir, 0o777) mkdir(es_data_dir) chmod_r(es_data_dir, 0o777) mkdir(es_tmp_dir) chmod_r(es_tmp_dir, 0o777) # start proxy and ES process proxy = start_proxy_for_service('elasticsearch', port, backend_port, update_listener, quiet=True, params={'protocol_version': 'HTTP/1.0'}) STATE['_proxy_'] = proxy if is_root(): cmd = "su localstack -c '%s'" % cmd thread = do_run(cmd, asynchronous, env_vars=env_vars) STATE['_thread_'] = thread return thread
def _check(fname, is_dir): test_entry = os.path.join(tmp_dir, fname) mkdir(test_entry) if is_dir else save_file(test_entry, "test content") assert not is_empty_dir(tmp_dir) assert is_empty_dir(tmp_dir, ignore_hidden=True) == (fname == ".hidden") rm_rf(test_entry) assert is_empty_dir(tmp_dir)
def install_local_kms(): local_arch = f"{platform.system().lower()}-{get_arch()}" binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", local_arch) if not os.path.exists(binary_path): log_install_msg("KMS") mkdir(INSTALL_DIR_KMS) kms_url = KMS_URL_PATTERN.replace("<arch>", local_arch) download(kms_url, binary_path) chmod_r(binary_path, 0o777)
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None): target_dir = os.path.join(INSTALL_DIR_INFRA, 'kinesis-mock') machine = platform.machine().lower() system = platform.system().lower() if machine == 'x86_64' or machine == 'amd64': if system == 'windows': target_file_name = 'kinesis-mock-mostly-static.exe' elif system == 'linux': target_file_name = 'kinesis-mock-linux-amd64-static' elif system == 'darwin': target_file_name = 'kinesis-mock-macos-amd64-dynamic' else: target_file_name = 'kinesis-mock.jar' else: target_file_name = 'kinesis-mock.jar' target_file = os.path.join(target_dir, target_file_name) if not os.path.exists(target_file): response = requests.get(KINESIS_MOCK_RELEASES) content = json.loads(to_str(response.content)) assets = content.get('assets', []) filtered = [x for x in assets if x['name'] == target_file_name] archive_url = filtered[0].get('browser_download_url') download(archive_url, target_file) port = port or config.PORT_KINESIS backend_port = get_free_tcp_port() kinesis_data_dir_param = '' if config.DATA_DIR: kinesis_data_dir = '%s/kinesis' % config.DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = 'SHOULD_PERSIST_DATA=true PERSIST_PATH=%s' % kinesis_data_dir if not config.LS_LOG: log_level = 'INFO' elif config.LS_LOG == 'warning': log_level = 'WARN' else: log_level = config.LS_LOG.upper log_level_param = 'LOG_LEVEL=%s' % (log_level) latency = config.KINESIS_LATENCY + 'ms' latency_param = 'CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s ' \ 'START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s ' \ 'DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s ' \ 'UPDATE_SHARD_COUNT_DURATION=%s' \ % (latency, latency, latency, latency, latency, latency, latency, latency, latency) if target_file_name.endswith('.jar'): cmd = 'KINESIS_MOCK_HTTP1_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s java -XX:+UseG1GC -jar %s' \ % (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, target_file) else: chmod_r(target_file, 0o777) cmd = 'KINESIS_MOCK_HTTP1_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s --gc=G1' \ % (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, target_file) start_proxy_for_service('kinesis', port, backend_port, update_listener) return do_run(cmd, asynchronous)
def install_elasticmq(): if not os.path.exists(INSTALL_DIR_ELASTICMQ): log_install_msg('ElasticMQ') mkdir(INSTALL_DIR_ELASTICMQ) # download archive tmp_archive = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar') if not os.path.exists(tmp_archive): download(ELASTICMQ_JAR_URL, tmp_archive) shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def install_elasticmq(): # TODO remove this function if we stop using ElasticMQ entirely if not os.path.exists(INSTALL_PATH_ELASTICMQ_JAR): log_install_msg("ElasticMQ") mkdir(INSTALL_DIR_ELASTICMQ) # download archive tmp_archive = os.path.join(config.dirs.tmp, "elasticmq-server.jar") if not os.path.exists(tmp_archive): download(ELASTICMQ_JAR_URL, tmp_archive) shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def install_local_kms(): local_arch = get_os() binary_path = INSTALL_PATH_KMS_BINARY_PATTERN.replace("<arch>", local_arch) if not os.path.exists(binary_path): log_install_msg("KMS") mkdir(INSTALL_DIR_KMS) # TODO ARM download platform specific binary kms_url = KMS_URL_PATTERN.replace("<arch>", local_arch) download(kms_url, binary_path) chmod_r(binary_path, 0o777)
def install_elasticmq(): if not os.path.exists(INSTALL_DIR_ELASTICMQ): LOGGER.info( 'Downloading and installing local ElasticMQ server. This may take some time.' ) mkdir(INSTALL_DIR_ELASTICMQ) # download archive if not os.path.exists(TMP_ARCHIVE_ELASTICMQ): download(ELASTICMQ_JAR_URL, TMP_ARCHIVE_ELASTICMQ) shutil.copy(TMP_ARCHIVE_ELASTICMQ, INSTALL_DIR_ELASTICMQ)
def get_lambda_code(func_name, retries=1, cache_time=None, env=None, region=None): if MOCK_OBJ: return "" env = aws_stack.get_environment(env) if cache_time is None and not aws_stack.is_local_env(env): cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT lambda_client = _connect("lambda", env=env, region=region) out = lambda_client.get_function(FunctionName=func_name) loc = out["Code"]["Location"] hash = md5(loc) folder = TMP_DOWNLOAD_FILE_PATTERN.replace("*", hash) filename = "archive.zip" archive = "%s/%s" % (folder, filename) try: mkdir(folder) if not os.path.isfile(archive): download(loc, archive, verify_ssl=False) if len(os.listdir(folder)) <= 1: zip_path = os.path.join(folder, filename) unzip(zip_path, folder) except Exception as e: print("WARN: %s" % e) rm_rf(archive) if retries > 0: return get_lambda_code(func_name, retries=retries - 1, cache_time=1, env=env) else: print("WARNING: Unable to retrieve lambda code: %s" % e) # traverse subdirectories and get script sources result = {} for root, subdirs, files in os.walk(folder): for file in files: prefix = root.split(folder)[-1] key = "%s/%s" % (prefix, file) if re.match(r".+\.py$", key) or re.match(r".+\.js$", key): codefile = "%s/%s" % (root, file) result[key] = load_file(codefile) # cleanup cache clean_cache( file_pattern=TMP_DOWNLOAD_FILE_PATTERN, last_clean_time=last_cache_cleanup_time, max_age=TMP_DOWNLOAD_CACHE_MAX_AGE, ) # TODO: delete only if cache_time is over rm_rf(folder) return result
def test_get_java_lib_folder_classpath_no_directories(self): base_dir = new_tmp_dir() jar_file = os.path.join(base_dir, "foo.jar") save_file(jar_file, "") lib_file = os.path.join(base_dir, "lib", "lib.jar") mkdir(os.path.dirname(lib_file)) save_file(lib_file, "") classpath = lambda_executors.Util.get_java_classpath(jar_file) self.assertIn(":foo.jar", classpath) self.assertIn("lib/lib.jar:", classpath) self.assertIn(":*.jar", classpath)
def install_kinesis_mock(): target_dir = INSTALL_PATH_KINESIS_MOCK machine = platform.machine().lower() system = platform.system().lower() version = platform.version().lower() is_probably_m1 = system == "darwin" and ("arm64" in version or "arm32" in version) LOG.debug("getting kinesis-mock for %s %s", system, machine) if is_env_true("KINESIS_MOCK_FORCE_JAVA"): # sometimes the static binaries may have problems, and we want to fal back to Java bin_file = "kinesis-mock.jar" elif (machine == "x86_64" or machine == "amd64") and not is_probably_m1: if system == "windows": bin_file = "kinesis-mock-mostly-static.exe" elif system == "linux": bin_file = "kinesis-mock-linux-amd64-static" elif system == "darwin": bin_file = "kinesis-mock-macos-amd64-dynamic" else: bin_file = "kinesis-mock.jar" else: bin_file = "kinesis-mock.jar" bin_file_path = os.path.join(target_dir, bin_file) if os.path.exists(bin_file_path): LOG.debug("kinesis-mock found at %s", bin_file_path) return bin_file_path response = requests.get(KINESIS_MOCK_RELEASE_URL) if not response.ok: raise ValueError("Could not get list of releases from %s: %s" % (KINESIS_MOCK_RELEASE_URL, response.text)) github_release = response.json() download_url = None for asset in github_release.get("assets", []): # find the correct binary in the release if asset["name"] == bin_file: download_url = asset["browser_download_url"] break if download_url is None: raise ValueError("could not find required binary %s in release %s" % (bin_file, KINESIS_MOCK_RELEASE_URL)) mkdir(target_dir) LOG.info("downloading kinesis-mock binary from %s", download_url) download(download_url, bin_file_path) chmod_r(bin_file_path, 0o777) return bin_file_path
def install_elasticsearch(): if not os.path.exists(INSTALL_DIR_ES): LOGGER.info('Downloading and installing local Elasticsearch server. This may take some time.') mkdir(INSTALL_DIR_INFRA) # download and extract archive download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, TMP_ARCHIVE_ES, INSTALL_DIR_INFRA) run('cd %s && mv elasticsearch* elasticsearch' % (INSTALL_DIR_INFRA)) for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'): dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777)
def install_elasticmq(): if SQS_BACKEND_IMPL != 'elasticmq': return # TODO remove this function if we stop using ElasticMQ entirely if not os.path.exists(INSTALL_PATH_ELASTICMQ_JAR): log_install_msg('ElasticMQ') mkdir(INSTALL_DIR_ELASTICMQ) # download archive tmp_archive = os.path.join(tempfile.gettempdir(), 'elasticmq-server.jar') if not os.path.exists(tmp_archive): download(ELASTICMQ_JAR_URL, tmp_archive) shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ)
def start_kinesis(port=PORT_KINESIS, asynchronous=False, shard_limit=100, update_listener=None): install.install_kinesalite() backend_port = DEFAULT_PORT_KINESIS_BACKEND kinesis_data_dir_param = '' if DATA_DIR: kinesis_data_dir = '%s/kinesis' % DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = '--path %s' % kinesis_data_dir cmd = ('%s/node_modules/kinesalite/cli.js --shardLimit %s --port %s %s' % (ROOT_PATH, shard_limit, backend_port, kinesis_data_dir_param)) print('Starting mock Kinesis (%s port %s)...' % (get_service_protocol(), port)) do_run(cmd, asynchronous) start_proxy_for_service('kinesis', port, backend_port, update_listener)
def start_dynamodb(port=PORT_DYNAMODB, asynchronous=False, update_listener=None): install.install_dynamodb_local() backend_port = DEFAULT_PORT_DYNAMODB_BACKEND ddb_data_dir_param = '-inMemory' if DATA_DIR: ddb_data_dir = '%s/dynamodb' % DATA_DIR mkdir(ddb_data_dir) ddb_data_dir_param = '-dbPath %s' % ddb_data_dir cmd = ('cd %s/infra/dynamodb/; java -Djava.library.path=./DynamoDBLocal_lib ' + '-jar DynamoDBLocal.jar -sharedDb -port %s %s') % (ROOT_PATH, backend_port, ddb_data_dir_param) print('Starting mock DynamoDB (%s port %s)...' % (get_service_protocol(), port)) start_proxy_for_service('dynamodb', port, backend_port, update_listener) return do_run(cmd, asynchronous)
def install_elasticsearch(): if not os.path.exists(INSTALL_DIR_ES): log_install_msg('Elasticsearch') mkdir(INSTALL_DIR_INFRA) # download and extract archive tmp_archive = os.path.join(tempfile.gettempdir(), 'localstack.es.zip') download_and_extract_with_retry(ELASTICSEARCH_JAR_URL, tmp_archive, INSTALL_DIR_INFRA) elasticsearch_dir = glob.glob( os.path.join(INSTALL_DIR_INFRA, 'elasticsearch*')) if not elasticsearch_dir: raise Exception('Unable to find Elasticsearch folder in %s' % INSTALL_DIR_INFRA) shutil.move(elasticsearch_dir[0], INSTALL_DIR_ES) for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'): dir_path = '%s/%s' % (INSTALL_DIR_ES, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # install default plugins for plugin in ELASTICSEARCH_PLUGIN_LIST: if is_alpine(): # https://github.com/pires/docker-elasticsearch/issues/56 os.environ['ES_TMPDIR'] = '/tmp' plugin_binary = os.path.join(INSTALL_DIR_ES, 'bin', 'elasticsearch-plugin') print('install elasticsearch-plugin %s' % (plugin)) run('%s install -b %s' % (plugin_binary, plugin)) # delete some plugins to free up space for plugin in ELASTICSEARCH_DELETE_MODULES: module_dir = os.path.join(INSTALL_DIR_ES, 'modules', plugin) rm_rf(module_dir) # disable x-pack-ml plugin (not working on Alpine) xpack_dir = os.path.join(INSTALL_DIR_ES, 'modules', 'x-pack-ml', 'platform') rm_rf(xpack_dir) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(INSTALL_DIR_ES, 'config', 'jvm.options') if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)', r'# \1', jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def install_elasticsearch(version=None): version = get_elasticsearch_install_version(version) install_dir = get_elasticsearch_install_dir(version) installed_executable = os.path.join(install_dir, 'bin', 'elasticsearch') if not os.path.exists(installed_executable): log_install_msg('Elasticsearch (%s)' % version) es_url = ELASTICSEARCH_URLS.get(version) if not es_url: raise Exception('Unable to find download URL for Elasticsearch version "%s"' % version) install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join(config.TMP_FOLDER, 'localstack.%s' % os.path.basename(es_url)) download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob(os.path.join(install_dir_parent, 'elasticsearch*')) if not elasticsearch_dir: raise Exception('Unable to find Elasticsearch folder in %s' % install_dir_parent) shutil.move(elasticsearch_dir[0], install_dir) for dir_name in ('data', 'logs', 'modules', 'plugins', 'config/scripts'): dir_path = os.path.join(install_dir, dir_name) mkdir(dir_path) chmod_r(dir_path, 0o777) # install default plugins for plugin in ELASTICSEARCH_PLUGIN_LIST: if is_alpine(): # https://github.com/pires/docker-elasticsearch/issues/56 os.environ['ES_TMPDIR'] = '/tmp' plugin_binary = os.path.join(install_dir, 'bin', 'elasticsearch-plugin') plugin_dir = os.path.join(install_dir, 'plugins', plugin) if not os.path.exists(plugin_dir): LOG.info('Installing Elasticsearch plugin %s' % (plugin)) run('%s install -b %s' % (plugin_binary, plugin)) # delete some plugins to free up space for plugin in ELASTICSEARCH_DELETE_MODULES: module_dir = os.path.join(install_dir, 'modules', plugin) rm_rf(module_dir) # disable x-pack-ml plugin (not working on Alpine) xpack_dir = os.path.join(install_dir, 'modules', 'x-pack-ml', 'platform') rm_rf(xpack_dir) # patch JVM options file - replace hardcoded heap size settings jvm_options_file = os.path.join(install_dir, 'config', 'jvm.options') if os.path.exists(jvm_options_file): jvm_options = load_file(jvm_options_file) jvm_options_replaced = re.sub(r'(^-Xm[sx][a-zA-Z0-9\.]+$)', r'# \1', jvm_options, flags=re.MULTILINE) if jvm_options != jvm_options_replaced: save_file(jvm_options_file, jvm_options_replaced)
def start_kinesis_mock(port=None, asynchronous=False, update_listener=None): kinesis_mock_bin = install.install_kinesis_mock() port = port or config.PORT_KINESIS backend_port = get_free_tcp_port() kinesis_data_dir_param = '' if config.DATA_DIR: kinesis_data_dir = '%s/kinesis' % config.DATA_DIR mkdir(kinesis_data_dir) # FIXME: workaround for https://github.com/localstack/localstack/issues/4227 streams_file = os.path.join(kinesis_data_dir, 'kinesis-data.json') if not os.path.exists(streams_file): with open(streams_file, 'w') as fd: fd.write('{"streams":{}}') kinesis_data_dir_param = 'SHOULD_PERSIST_DATA=true PERSIST_PATH=%s' % kinesis_data_dir if not config.LS_LOG: log_level = 'INFO' elif config.LS_LOG == 'warning': log_level = 'WARN' else: log_level = config.LS_LOG.upper() log_level_param = 'LOG_LEVEL=%s' % log_level latency = config.KINESIS_LATENCY + 'ms' latency_param = 'CREATE_STREAM_DURATION=%s DELETE_STREAM_DURATION=%s REGISTER_STREAM_CONSUMER_DURATION=%s ' \ 'START_STREAM_ENCRYPTION_DURATION=%s STOP_STREAM_ENCRYPTION_DURATION=%s ' \ 'DEREGISTER_STREAM_CONSUMER_DURATION=%s MERGE_SHARDS_DURATION=%s SPLIT_SHARD_DURATION=%s ' \ 'UPDATE_SHARD_COUNT_DURATION=%s' \ % (latency, latency, latency, latency, latency, latency, latency, latency, latency) if config.KINESIS_INITIALIZE_STREAMS != '': initialize_streams_param = 'INITIALIZE_STREAMS=%s' % ( config.KINESIS_INITIALIZE_STREAMS) else: initialize_streams_param = '' if kinesis_mock_bin.endswith('.jar'): cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s java -XX:+UseG1GC -jar %s' \ % (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, initialize_streams_param, kinesis_mock_bin) else: chmod_r(kinesis_mock_bin, 0o777) cmd = 'KINESIS_MOCK_PLAIN_PORT=%s SHARD_LIMIT=%s %s %s %s %s %s --gc=G1' \ % (backend_port, config.KINESIS_SHARD_LIMIT, latency_param, kinesis_data_dir_param, log_level_param, initialize_streams_param, kinesis_mock_bin) LOGGER.info('starting kinesis-mock proxy %d:%d with cmd: %s', port, backend_port, cmd) start_proxy_for_service('kinesis', port, backend_port, update_listener) return do_run(cmd, asynchronous)
def install_amazon_kinesis_client_libs(): # install KCL/STS JAR files if not os.path.exists(INSTALL_DIR_KCL): mkdir(INSTALL_DIR_KCL) if not os.path.exists(TMP_ARCHIVE_STS): download(STS_JAR_URL, TMP_ARCHIVE_STS) shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL) # Compile Java files from localstack.utils.kinesis import kclipy_helper classpath = kclipy_helper.get_kcl_classpath() java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH if not glob.glob(class_files): run('javac -cp "%s" %s' % (classpath, java_files))
def create_lambda_archive(script, get_content=False, libs=[], runtime=None, file_name=None): """Utility method to create a Lambda function archive""" runtime = runtime or LAMBDA_DEFAULT_RUNTIME tmp_dir = tempfile.mkdtemp(prefix=ARCHIVE_DIR_PREFIX) TMP_FILES.append(tmp_dir) file_name = file_name or get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER, runtime=runtime) script_file = os.path.join(tmp_dir, file_name) if os.path.sep in script_file: mkdir(os.path.dirname(script_file)) # create __init__.py files along the path to allow Python imports path = file_name.split(os.path.sep) for i in range(1, len(path)): save_file(os.path.join(tmp_dir, *(path[:i] + ["__init__.py"])), "") save_file(script_file, script) chmod_r(script_file, 0o777) # copy libs for lib in libs: paths = [lib, "%s.py" % lib] try: module = importlib.import_module(lib) paths.append(module.__file__) except Exception: pass target_dir = tmp_dir root_folder = os.path.join(LOCALSTACK_VENV_FOLDER, "lib/python*/site-packages") if lib == "localstack": paths = ["localstack/*.py", "localstack/utils"] root_folder = LOCALSTACK_ROOT_FOLDER target_dir = os.path.join(tmp_dir, lib) mkdir(target_dir) for path in paths: file_to_copy = path if path.startswith("/") else os.path.join( root_folder, path) for file_path in glob.glob(file_to_copy): name = os.path.join(target_dir, file_path.split(os.path.sep)[-1]) if os.path.isdir(file_path): copy_dir(file_path, name) else: shutil.copyfile(file_path, name) # create zip file result = create_zip_file(tmp_dir, get_content=get_content) return result
def install_amazon_kinesis_client_libs(): # install KCL/STS JAR files if not os.path.exists(INSTALL_DIR_KCL): mkdir(INSTALL_DIR_KCL) tmp_archive = os.path.join(tempfile.gettempdir(), 'aws-java-sdk-sts.jar') if not os.path.exists(tmp_archive): download(STS_JAR_URL, tmp_archive) shutil.copy(tmp_archive, INSTALL_DIR_KCL) # Compile Java files from localstack.utils.kinesis import kclipy_helper classpath = kclipy_helper.get_kcl_classpath() java_files = '%s/utils/kinesis/java/cloud/localstack/*.java' % ROOT_PATH class_files = '%s/utils/kinesis/java/cloud/localstack/*.class' % ROOT_PATH
def get_lambda_code(func_name, retries=1, cache_time=None, env=None): if MOCK_OBJ: return '' env = aws_stack.get_environment(env) if cache_time is None and not aws_stack.is_local_env(env): cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT out = cmd_lambda('get-function --function-name %s' % func_name, env, cache_time) out = json.loads(out) loc = out['Code']['Location'] hash = md5(loc) folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash) filename = 'archive.zip' archive = '%s/%s' % (folder, filename) try: mkdir(folder) if not os.path.isfile(archive): download(loc, archive, verify_ssl=False) if len(os.listdir(folder)) <= 1: zip_path = os.path.join(folder, filename) unzip(zip_path, folder) except Exception as e: print('WARN: %s' % e) rm_rf(archive) if retries > 0: return get_lambda_code(func_name, retries=retries - 1, cache_time=1, env=env) else: print('WARNING: Unable to retrieve lambda code: %s' % e) # traverse subdirectories and get script sources result = {} for root, subdirs, files in os.walk(folder): for file in files: prefix = root.split(folder)[-1] key = '%s/%s' % (prefix, file) if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key): codefile = '%s/%s' % (root, file) result[key] = load_file(codefile) # cleanup cache clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN, last_clean_time=last_cache_cleanup_time, max_age=TMP_DOWNLOAD_CACHE_MAX_AGE) # TODO: delete only if cache_time is over rm_rf(folder) return result
def install_amazon_kinesis_client_libs(): # install KCL/STS JAR files if not os.path.exists(INSTALL_DIR_KCL): mkdir(INSTALL_DIR_KCL) if not os.path.exists(TMP_ARCHIVE_STS): download(STS_JAR_URL, TMP_ARCHIVE_STS) shutil.copy(TMP_ARCHIVE_STS, INSTALL_DIR_KCL) # Compile Java files from localstack.utils.kinesis import kclipy_helper classpath = kclipy_helper.get_kcl_classpath() java_files = '%s/utils/kinesis/java/com/atlassian/*.java' % ROOT_PATH class_files = '%s/utils/kinesis/java/com/atlassian/*.class' % ROOT_PATH if not glob.glob(class_files): run('javac -cp "%s" %s' % (classpath, java_files))
def get_lambda_code(func_name, retries=1, cache_time=None, env=None): if MOCK_OBJ: return '' env = aws_stack.get_environment(env) if cache_time is None and env.region != REGION_LOCAL: cache_time = AWS_LAMBDA_CODE_CACHE_TIMEOUT out = cmd_lambda('get-function --function-name %s' % func_name, env, cache_time) out = json.loads(out) loc = out['Code']['Location'] hash = md5(loc) folder = TMP_DOWNLOAD_FILE_PATTERN.replace('*', hash) filename = 'archive.zip' archive = '%s/%s' % (folder, filename) try: mkdir(folder) if not os.path.isfile(archive): download(loc, archive, verify_ssl=False) if len(os.listdir(folder)) <= 1: zip_path = os.path.join(folder, filename) unzip(zip_path, folder) except Exception as e: print('WARN: %s' % e) rm_rf(archive) if retries > 0: return get_lambda_code(func_name, retries=retries - 1, cache_time=1, env=env) else: print('WARNING: Unable to retrieve lambda code: %s' % e) # traverse subdirectories and get script sources result = {} for root, subdirs, files in os.walk(folder): for file in files: prefix = root.split(folder)[-1] key = '%s/%s' % (prefix, file) if re.match(r'.+\.py$', key) or re.match(r'.+\.js$', key): codefile = '%s/%s' % (root, file) result[key] = load_file(codefile) # cleanup cache clean_cache(file_pattern=TMP_DOWNLOAD_FILE_PATTERN, last_clean_time=last_cache_cleanup_time, max_age=TMP_DOWNLOAD_CACHE_MAX_AGE) # TODO: delete only if cache_time is over rm_rf(folder) return result
def create_lambda_archive(script, stream=None, get_content=False, libs=[], runtime=None): """Utility method to create a Lambda function archive""" tmp_dir = tempfile.mkdtemp(prefix=ARCHIVE_DIR_PREFIX) TMP_FILES.append(tmp_dir) file_name = get_handler_file_from_name(LAMBDA_DEFAULT_HANDLER, runtime=runtime) script_file = '%s/%s' % (tmp_dir, file_name) save_file(script_file, script) # copy libs for lib in libs: paths = [lib, '%s.py' % lib] target_dir = tmp_dir root_folder = '%s/lib/python*/site-packages' % LOCALSTACK_VENV_FOLDER if lib == 'localstack': paths = ['localstack/*.py', 'localstack/utils'] root_folder = LOCALSTACK_ROOT_FOLDER target_dir = '%s/%s/' % (tmp_dir, lib) mkdir(target_dir) for path in paths: file_to_copy = '%s/%s' % (root_folder, path) for file_path in glob.glob(file_to_copy): run('cp -r %s %s/' % (file_path, target_dir)) # create zip file return create_zip_file(tmp_dir, get_content=True)
def test_lambda_runtimes(): lambda_client = aws_stack.connect_to_service('lambda') # deploy and invoke lambda - Python 2.7 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON27) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON27) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' if use_docker(): # deploy and invoke lambda - Python 3.6 zip_file = testutil.create_lambda_archive(load_file(TEST_LAMBDA_PYTHON3), get_content=True, libs=TEST_LAMBDA_LIBS, runtime=LAMBDA_RUNTIME_PYTHON36) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_PY3, zip_file=zip_file, runtime=LAMBDA_RUNTIME_PYTHON36) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_PY3, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java if not os.path.exists(TEST_LAMBDA_JAVA): mkdir(os.path.dirname(TEST_LAMBDA_JAVA)) download(TEST_LAMBDA_JAR_URL, TEST_LAMBDA_JAVA) zip_file = testutil.create_zip_file(TEST_LAMBDA_JAVA, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'LinkedHashMap' in to_str(result_data) # test SNSEvent result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"Sns": {"Message": "{}"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'async': 'True'} # test DDBEvent result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, InvocationType='Event', Payload=b'{"Records": [{"dynamodb": {"Message": "{}"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'async': 'True'} # test KinesisEvent result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA, Payload=b'{"Records": [{"Kinesis": {"Data": "data", "PartitionKey": "partition"}}]}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert 'KinesisEvent' in to_str(result_data) # deploy and invoke lambda - Java with stream handler testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_STREAM, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.LambdaStreamHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_STREAM, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke lambda - Java with serializable input object testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, zip_file=zip_file, runtime=LAMBDA_RUNTIME_JAVA8, handler='cloud.localstack.sample.SerializedInputLambdaHandler') result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JAVA_SERIALIZABLE, Payload=b'{"bucket": "test_bucket", "key": "test_key"}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert json.loads(to_str(result_data)) == {'validated': True, 'bucket': 'test_bucket', 'key': 'test_key'} if use_docker(): # deploy and invoke lambda - Node.js zip_file = testutil.create_zip_file(TEST_LAMBDA_NODEJS, get_content=True) testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_JS, zip_file=zip_file, handler='lambda_integration.handler', runtime=LAMBDA_RUNTIME_NODEJS) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_JS, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}' # deploy and invoke - .NET Core 2.0. Its already a zip zip_file = TEST_LAMBDA_DOTNETCORE2 zip_file_content = None with open(zip_file, 'rb') as file_obj: zip_file_content = file_obj.read() testutil.create_lambda_function(func_name=TEST_LAMBDA_NAME_DOTNETCORE2, zip_file=zip_file_content, handler='DotNetCore2::DotNetCore2.Lambda.Function::SimpleFunctionHandler', runtime=LAMBDA_RUNTIME_DOTNETCORE2) result = lambda_client.invoke(FunctionName=TEST_LAMBDA_NAME_DOTNETCORE2, Payload=b'{}') assert result['StatusCode'] == 200 result_data = result['Payload'].read() assert to_str(result_data).strip() == '{}'
def check_dynamodb(expect_shutdown=False, print_error=False): out = None try: # wait for port to be opened wait_for_port_open(DEFAULT_PORT_DYNAMODB_BACKEND) # check DynamoDB out = aws_stack.connect_to_service(service_name='dynamodb').list_tables() except Exception as e: if print_error: LOGGER.error('DynamoDB health check failed: %s %s' % (e, traceback.format_exc())) if expect_shutdown: assert out is None else: assert isinstance(out['TableNames'], list) def start_dynamodb(port=PORT_DYNAMODB, async=False, update_listener=None): install.install_dynamodb_local() backend_port = DEFAULT_PORT_DYNAMODB_BACKEND ddb_data_dir_param = '-inMemory' if DATA_DIR: ddb_data_dir = '%s/dynamodb' % DATA_DIR mkdir(ddb_data_dir) ddb_data_dir_param = '-dbPath %s' % ddb_data_dir cmd = ('cd %s/infra/dynamodb/; java -Djava.library.path=./DynamoDBLocal_lib ' + '-jar DynamoDBLocal.jar -sharedDb -port %s %s') % (ROOT_PATH, backend_port, ddb_data_dir_param) print('Starting mock DynamoDB (%s port %s)...' % (get_service_protocol(), port)) start_proxy_for_service('dynamodb', port, backend_port, update_listener) return do_run(cmd, async)
def start_infra_in_docker(): # load plugins before starting the docker container plugin_configs = load_plugins() plugin_run_params = ' '.join([ entry.get('docker', {}).get('run_flags', '') for entry in plugin_configs]) services = os.environ.get('SERVICES', '') entrypoint = os.environ.get('ENTRYPOINT', '') cmd = os.environ.get('CMD', '') image_name = os.environ.get('IMAGE_NAME', constants.DOCKER_IMAGE_NAME) service_ports = config.SERVICE_PORTS force_noninteractive = os.environ.get('FORCE_NONINTERACTIVE', '') # construct port mappings ports_list = sorted(service_ports.values()) start_port = 0 last_port = 0 port_ranges = [] for i in range(0, len(ports_list)): if not start_port: start_port = ports_list[i] if not last_port: last_port = ports_list[i] if ports_list[i] > last_port + 1: port_ranges.append([start_port, last_port]) start_port = ports_list[i] elif i >= len(ports_list) - 1: port_ranges.append([start_port, ports_list[i]]) last_port = ports_list[i] port_mappings = ' '.join( '-p {start}-{end}:{start}-{end}'.format(start=entry[0], end=entry[1]) if entry[0] < entry[1] else '-p {port}:{port}'.format(port=entry[0]) for entry in port_ranges) if services: port_mappings = '' for service, port in service_ports.items(): port_mappings += ' -p {port}:{port}'.format(port=port) env_str = '' for env_var in config.CONFIG_ENV_VARS: value = os.environ.get(env_var, None) if value is not None: env_str += '-e %s="%s" ' % (env_var, value) data_dir_mount = '' data_dir = os.environ.get('DATA_DIR', None) if data_dir is not None: container_data_dir = '/tmp/localstack_data' data_dir_mount = '-v "%s:%s" ' % (data_dir, container_data_dir) env_str += '-e DATA_DIR="%s" ' % container_data_dir interactive = '' if force_noninteractive or in_ci() else '-it ' # append space if parameter is set entrypoint = '%s ' % entrypoint if entrypoint else entrypoint plugin_run_params = '%s ' % plugin_run_params if plugin_run_params else plugin_run_params docker_cmd = ('docker run %s%s%s%s' + '-p 8080:8080 %s %s' + '-v "%s:/tmp/localstack" -v "%s:%s" ' + '-e DOCKER_HOST="unix://%s" ' + '-e HOST_TMP_FOLDER="%s" "%s" %s') % ( interactive, entrypoint, env_str, plugin_run_params, port_mappings, data_dir_mount, config.TMP_FOLDER, config.DOCKER_SOCK, config.DOCKER_SOCK, config.DOCKER_SOCK, config.HOST_TMP_FOLDER, image_name, cmd ) mkdir(config.TMP_FOLDER) run_cmd_safe(cmd='chmod -R 777 "%s"' % config.TMP_FOLDER) print(docker_cmd) t = ShellCommandThread(docker_cmd, outfile=subprocess.PIPE) t.start() time.sleep(2) t.process.wait() sys.exit(t.process.returncode)
def set_function_code(code, lambda_name): def generic_handler(event, context): raise Exception(('Unable to find executor for Lambda function "%s". ' + 'Note that Node.js and .NET Core Lambdas currently require LAMBDA_EXECUTOR=docker') % lambda_name) lambda_handler = generic_handler lambda_cwd = None arn = func_arn(lambda_name) runtime = arn_to_lambda[arn].runtime handler_name = arn_to_lambda.get(arn).handler lambda_environment = arn_to_lambda.get(arn).envvars if not handler_name: handler_name = LAMBDA_DEFAULT_HANDLER handler_file = get_handler_file_from_name(handler_name, runtime=runtime) handler_function = get_handler_function_from_name(handler_name, runtime=runtime) # Stop/remove any containers that this arn uses. LAMBDA_EXECUTOR.cleanup(arn) if 'S3Bucket' in code: s3_client = aws_stack.connect_to_service('s3') bytes_io = BytesIO() try: s3_client.download_fileobj(code['S3Bucket'], code['S3Key'], bytes_io) zip_file_content = bytes_io.getvalue() except Exception as e: return error_response('Unable to fetch Lambda archive from S3: %s' % e, 404) elif 'ZipFile' in code: zip_file_content = code['ZipFile'] zip_file_content = base64.b64decode(zip_file_content) else: return error_response('No valid Lambda archive specified.', 400) # save tmp file tmp_dir = '%s/zipfile.%s' % (config.TMP_FOLDER, short_uid()) mkdir(tmp_dir) tmp_file = '%s/%s' % (tmp_dir, LAMBDA_ZIP_FILE_NAME) save_file(tmp_file, zip_file_content) TMP_FILES.append(tmp_dir) lambda_cwd = tmp_dir # check if this is a ZIP file is_zip = is_zip_file(zip_file_content) if is_zip: unzip(tmp_file, tmp_dir) main_file = '%s/%s' % (tmp_dir, handler_file) if not os.path.isfile(main_file): # check if this is a zip file that contains a single JAR file jar_files = glob.glob('%s/*.jar' % tmp_dir) if len(jar_files) == 1: main_file = jar_files[0] if os.path.isfile(main_file): # make sure the file is actually readable, then read contents ensure_readable(main_file) with open(main_file, 'rb') as file_obj: zip_file_content = file_obj.read() else: file_list = run('ls -la %s' % tmp_dir) LOG.debug('Lambda archive content:\n%s' % file_list) return error_response('Unable to find handler script in Lambda archive.', 400, error_type='ValidationError') # it could be a JAR file (regardless of whether wrapped in a ZIP file or not) is_jar = is_jar_archive(zip_file_content) if is_jar: def execute(event, context): result, log_output = lambda_executors.EXECUTOR_LOCAL.execute_java_lambda(event, context, handler=arn_to_lambda[arn].handler, main_file=main_file) return result lambda_handler = execute elif runtime.startswith('python') and not use_docker(): try: lambda_handler = exec_lambda_code(zip_file_content, handler_function=handler_function, lambda_cwd=lambda_cwd, lambda_env=lambda_environment) except Exception as e: raise Exception('Unable to get handler function from lambda code.', e) if not is_zip and not is_jar: raise Exception('Uploaded Lambda code is neither a ZIP nor JAR file.') add_function_mapping(lambda_name, lambda_handler, lambda_cwd) return {'FunctionName': lambda_name}
from localstack.utils.aws import aws_stack from localstack.utils.common import mkdir from localstack.services import install from localstack.services.infra import get_service_protocol, start_proxy_for_service, do_run from localstack.services.install import ROOT_PATH LOGGER = logging.getLogger(__name__) def start_kinesis(port=PORT_KINESIS, async=False, shard_limit=100, update_listener=None): install.install_kinesalite() backend_port = DEFAULT_PORT_KINESIS_BACKEND kinesis_data_dir_param = '' if DATA_DIR: kinesis_data_dir = '%s/kinesis' % DATA_DIR mkdir(kinesis_data_dir) kinesis_data_dir_param = '--path %s' % kinesis_data_dir cmd = ('%s/node_modules/kinesalite/cli.js --shardLimit %s --port %s %s' % (ROOT_PATH, shard_limit, backend_port, kinesis_data_dir_param)) print('Starting mock Kinesis (%s port %s)...' % (get_service_protocol(), port)) start_proxy_for_service('kinesis', port, backend_port, update_listener) return do_run(cmd, async) def check_kinesis(expect_shutdown=False, print_error=False): out = None try: # check Kinesis out = aws_stack.connect_to_service(service_name='kinesis').list_streams() except Exception as e: if print_error: