def run_app_sync(*args, loop=None, shutdown_event=None):
kwargs = {}
config = Config()
cert_file_name, key_file_name = ssl_creds or (None, None)
if cert_file_name:
kwargs["certfile"] = cert_file_name
config.certfile = cert_file_name
if key_file_name:
kwargs["keyfile"] = key_file_name
config.keyfile = key_file_name
setup_quart_logging()
config.bind = [f"{bind_address}:{port}" for bind_address in bind_addresses]
config.workers = len(bind_addresses)
loop = loop or ensure_event_loop()
run_kwargs = {}
if shutdown_event:
run_kwargs["shutdown_trigger"] = shutdown_event.wait
try:
try:
return loop.run_until_complete(serve(app, config, **run_kwargs))
except Exception as e:
LOG.info(
"Error running server event loop on port %s: %s %s",
port,
e,
traceback.format_exc(),
)
if "SSL" in str(e):
c_exists = os.path.exists(cert_file_name)
k_exists = os.path.exists(key_file_name)
c_size = len(load_file(cert_file_name)) if c_exists else 0
k_size = len(load_file(key_file_name)) if k_exists else 0
LOG.warning(
"Unable to create SSL context. Cert files exist: %s %s (%sB), %s %s (%sB)",
cert_file_name,
c_exists,
c_size,
key_file_name,
k_exists,
k_size,
)
raise
finally:
try:
_cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
finally:
asyncio.set_event_loop(None)
loop.close()
def on_get(self, request):
from localstack.utils.aws.aws_stack import get_valid_regions
deploy_html_file = os.path.join(constants.MODULE_MAIN_PATH, "services",
"cloudformation", "deploy.html")
deploy_html = load_file(deploy_html_file)
req_params = request.values
params = {
"stackName": "stack1",
"templateBody": "{}",
"errorMessage": "''",
"regions": json.dumps(sorted(list(get_valid_regions()))),
}
download_url = req_params.get("templateURL")
if download_url:
try:
LOG.debug(
"Attempting to download CloudFormation template URL: %s",
download_url)
template_body = requests.get(download_url).text
template_body = parse_json_or_yaml(template_body)
params["templateBody"] = json.dumps(template_body)
except Exception as e:
msg = f"Unable to download CloudFormation template URL: {e}"
LOG.info(msg)
params["errorMessage"] = json.dumps(msg.replace("\n", " - "))
# using simple string replacement here, for simplicity (could be replaced with, e.g., jinja)
for key, value in params.items():
deploy_html = deploy_html.replace(f"<{key}>", value)
return Response(deploy_html, mimetype="text/html")
def test_cdk_bootstrap_redeploy(self, is_change_set_finished,
cleanup_stacks, cleanup_changesets):
"""Test that simulates a sequence of commands executed by CDK when running 'cdk bootstrap' twice"""
base_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"..")
requests_file = os.path.join(base_folder, "files",
"cdk-bootstrap-requests.json")
operations = json.loads(load_file(requests_file))
change_set_name = "cdk-deploy-change-set-a4b98b18"
stack_name = "CDKToolkit-a4b98b18"
try:
headers = aws_stack.mock_aws_request_headers("cloudformation")
base_url = config.get_edge_url()
for op in operations:
url = f"{base_url}{op['path']}"
data = op["data"]
requests.request(method=op["method"],
url=url,
headers=headers,
data=data)
if "Action=ExecuteChangeSet" in data:
assert wait_until(is_change_set_finished(change_set_name),
_max_wait=20,
strategy="linear")
finally:
# clean up
cleanup_changesets([change_set_name])
cleanup_stacks([stack_name])
def get_service_action_names(service: str, version: str = None) -> Set[str]:
"""Returns, for a given service name and version, the list of available service action names."""
version = version or DEFAULT_SERVICE_VERSIONS.get(service)
key = f"{service}:{version}"
result = SERVICE_ACTIONS_CACHE.get(key)
if not result:
file_path = os.path.join(os.path.dirname(botocore.__file__), "data",
service, version, "service-2.json")
content = json.loads(to_str(load_file(file_path)) or "{}")
result = set(content.get("operations", {}).keys())
SERVICE_ACTIONS_CACHE[key] = result
return result
def install_opensearch(version=None):
# locally import to avoid having a dependency on ASF when starting the CLI
from localstack.aws.api.opensearch import EngineType
from localstack.services.opensearch import versions
if not version:
version = OPENSEARCH_DEFAULT_VERSION
version = get_opensearch_install_version(version)
install_dir = get_opensearch_install_dir(version)
installed_executable = os.path.join(install_dir, "bin", "opensearch")
if not os.path.exists(installed_executable):
with OS_INSTALL_LOCKS.setdefault(version, threading.Lock()):
if not os.path.exists(installed_executable):
log_install_msg(f"OpenSearch ({version})")
opensearch_url = versions.get_download_url(
version, EngineType.OpenSearch)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(
config.dirs.tmp,
f"localstack.{os.path.basename(opensearch_url)}")
download_and_extract_with_retry(opensearch_url, tmp_archive,
install_dir_parent)
opensearch_dir = glob.glob(
os.path.join(install_dir_parent, "opensearch*"))
if not opensearch_dir:
raise Exception(
f"Unable to find OpenSearch folder in {install_dir_parent}"
)
shutil.move(opensearch_dir[0], install_dir)
for dir_name in ("data", "logs", "modules", "plugins",
"config/scripts"):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, "config", "jvm.options")
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)",
r"# \1",
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def test_export_oas3_openapi(apigateway_client):
spec_file = load_file(TEST_IMPORT_PETSTORE_SWAGGER)
response = apigateway_client.import_rest_api(failOnWarnings=True,
body=spec_file)
assert response.get("ResponseMetadata").get("HTTPStatusCode") == 201
response = apigateway_client.get_export(restApiId=response["id"],
stageName="local",
exportType="oas3")
spec_object = json.loads(response["body"].read())
# required keys
expected_keys = [
"openapi",
"info",
]
assert all(k in spec_object.keys() for k in expected_keys)
assert spec_object["info"]["title"] == "PetStore"
assert spec_object["info"]["version"] is not None
# optional keys
optional_keys = ["paths"]
assert all(k in spec_object.keys() for k in optional_keys)
def install_dynamodb_local():
if not os.path.exists(INSTALL_PATH_DDB_JAR):
log_install_msg("DynamoDB")
# download and extract archive
tmp_archive = os.path.join(tempfile.gettempdir(), "localstack.ddb.zip")
download_and_extract_with_retry(DYNAMODB_JAR_URL, tmp_archive,
INSTALL_DIR_DDB)
# fix logging configuration for DynamoDBLocal
log4j2_config = """<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN"><AppenderRef ref="Console"/></Root>
</Loggers>
</Configuration>"""
log4j2_file = os.path.join(INSTALL_DIR_DDB, "log4j2.xml")
save_file(log4j2_file, log4j2_config)
run_safe(lambda: run(["zip", "-u", "DynamoDBLocal.jar", "log4j2.xml"],
cwd=INSTALL_DIR_DDB))
# download agent JAR
if not os.path.exists(DDB_AGENT_JAR_PATH):
download(DDB_AGENT_JAR_URL, DDB_AGENT_JAR_PATH)
if not os.path.exists(JAVASSIST_JAR_PATH):
download(JAVASSIST_JAR_URL, JAVASSIST_JAR_PATH)
# ensure that javassist.jar is in the manifest classpath
run(["unzip", "-o", "DynamoDBLocal.jar", "META-INF/MANIFEST.MF"],
cwd=INSTALL_DIR_DDB)
manifest_file = os.path.join(INSTALL_DIR_DDB, "META-INF", "MANIFEST.MF")
manifest = load_file(manifest_file)
if "javassist.jar" not in manifest:
manifest = manifest.replace("Class-Path:", "Class-Path: javassist.jar",
1)
save_file(manifest_file, manifest)
run(["zip", "-u", "DynamoDBLocal.jar", "META-INF/MANIFEST.MF"],
cwd=INSTALL_DIR_DDB)
def create_lambda_function(
func_name,
zip_file=None,
event_source_arn=None,
handler_file=None,
handler=None,
starting_position=None,
runtime=None,
envvars=None,
tags=None,
libs=None,
delete=False,
layers=None,
client=None,
role=None,
timeout=None,
region_name=None,
**kwargs,
):
"""Utility method to create a new function via the Lambda API"""
if envvars is None:
envvars = {}
if tags is None:
tags = {}
if libs is None:
libs = []
starting_position = starting_position or LAMBDA_DEFAULT_STARTING_POSITION
runtime = runtime or LAMBDA_DEFAULT_RUNTIME
client = client or aws_stack.connect_to_service("lambda",
region_name=region_name)
# load zip file content if handler_file is specified
if not zip_file and handler_file:
file_content = load_file(handler_file) if os.path.exists(
handler_file) else handler_file
if libs or not handler:
zip_file = create_lambda_archive(
file_content,
libs=libs,
get_content=True,
runtime=runtime or LAMBDA_DEFAULT_RUNTIME,
)
else:
zip_file = create_zip_file(handler_file, get_content=True)
handler = handler or LAMBDA_DEFAULT_HANDLER
if delete:
try:
# Delete function if one already exists
client.delete_function(FunctionName=func_name)
except Exception:
pass
lambda_code = {"ZipFile": zip_file}
if len(zip_file) > MAX_LAMBDA_ARCHIVE_UPLOAD_SIZE:
s3 = aws_stack.connect_to_service("s3")
aws_stack.get_or_create_bucket(LAMBDA_ASSETS_BUCKET_NAME)
asset_key = f"{short_uid()}.zip"
s3.upload_fileobj(Fileobj=io.BytesIO(zip_file),
Bucket=LAMBDA_ASSETS_BUCKET_NAME,
Key=asset_key)
lambda_code = {
"S3Bucket": LAMBDA_ASSETS_BUCKET_NAME,
"S3Key": asset_key
}
# create function
additional_kwargs = kwargs
kwargs = {
"FunctionName": func_name,
"Runtime": runtime,
"Handler": handler,
"Role": role or LAMBDA_TEST_ROLE,
"Code": lambda_code,
"Timeout": timeout or LAMBDA_TIMEOUT_SEC,
"Environment": dict(Variables=envvars),
"Tags": tags,
}
kwargs.update(additional_kwargs)
if layers:
kwargs["Layers"] = layers
create_func_resp = client.create_function(**kwargs)
resp = {
"CreateFunctionResponse": create_func_resp,
"CreateEventSourceMappingResponse": None,
}
# create event source mapping
if event_source_arn:
resp[
"CreateEventSourceMappingResponse"] = client.create_event_source_mapping(
FunctionName=func_name,
EventSourceArn=event_source_arn,
StartingPosition=starting_position,
)
return resp
def install_elasticsearch(version=None):
# locally import to avoid having a dependency on ASF when starting the CLI
from localstack.aws.api.opensearch import EngineType
from localstack.services.opensearch import versions
if not version:
version = ELASTICSEARCH_DEFAULT_VERSION
version = get_elasticsearch_install_version(version)
install_dir = get_elasticsearch_install_dir(version)
installed_executable = os.path.join(install_dir, "bin", "elasticsearch")
if not os.path.exists(installed_executable):
log_install_msg(f"Elasticsearch ({version})")
es_url = versions.get_download_url(version, EngineType.Elasticsearch)
install_dir_parent = os.path.dirname(install_dir)
mkdir(install_dir_parent)
# download and extract archive
tmp_archive = os.path.join(config.dirs.tmp,
f"localstack.{os.path.basename(es_url)}")
download_and_extract_with_retry(es_url, tmp_archive,
install_dir_parent)
elasticsearch_dir = glob.glob(
os.path.join(install_dir_parent, "elasticsearch*"))
if not elasticsearch_dir:
raise Exception(
f"Unable to find Elasticsearch folder in {install_dir_parent}")
shutil.move(elasticsearch_dir[0], install_dir)
for dir_name in ("data", "logs", "modules", "plugins",
"config/scripts"):
dir_path = os.path.join(install_dir, dir_name)
mkdir(dir_path)
chmod_r(dir_path, 0o777)
# install default plugins
for plugin in ELASTICSEARCH_PLUGIN_LIST:
plugin_binary = os.path.join(install_dir, "bin",
"elasticsearch-plugin")
plugin_dir = os.path.join(install_dir, "plugins", plugin)
if not os.path.exists(plugin_dir):
LOG.info("Installing Elasticsearch plugin %s", plugin)
def try_install():
output = run([plugin_binary, "install", "-b", plugin])
LOG.debug("Plugin installation output: %s", output)
# We're occasionally seeing javax.net.ssl.SSLHandshakeException -> add download retries
download_attempts = 3
try:
retry(try_install, retries=download_attempts - 1, sleep=2)
except Exception:
LOG.warning(
"Unable to download Elasticsearch plugin '%s' after %s attempts",
plugin,
download_attempts,
)
if not os.environ.get("IGNORE_ES_DOWNLOAD_ERRORS"):
raise
# delete some plugins to free up space
for plugin in ELASTICSEARCH_DELETE_MODULES:
module_dir = os.path.join(install_dir, "modules", plugin)
rm_rf(module_dir)
# disable x-pack-ml plugin (not working on Alpine)
xpack_dir = os.path.join(install_dir, "modules", "x-pack-ml", "platform")
rm_rf(xpack_dir)
# patch JVM options file - replace hardcoded heap size settings
jvm_options_file = os.path.join(install_dir, "config", "jvm.options")
if os.path.exists(jvm_options_file):
jvm_options = load_file(jvm_options_file)
jvm_options_replaced = re.sub(r"(^-Xm[sx][a-zA-Z0-9.]+$)",
r"# \1",
jvm_options,
flags=re.MULTILINE)
if jvm_options != jvm_options_replaced:
save_file(jvm_options_file, jvm_options_replaced)
def get_host_kernel_version() -> str:
return load_file("/proc/version", "failed").strip()
def is_redhat() -> bool:
from localstack.utils.files import load_file
return "rhel" in load_file("/etc/os-release", "")
def is_debian() -> bool:
from localstack.utils.files import load_file
return "Debian" in load_file("/etc/issue", "")
