def get_image_environment_variable(env_name: str) -> Optional[str]:
image_name = get_docker_image_to_start()
image_info = DOCKER_CLIENT.inspect_image(image_name)
image_envs = image_info["Config"]["Env"]
try:
found_env = next(env for env in image_envs if env.startswith(env_name))
except StopIteration:
return None
return found_env.split("=")[1]
def get_main_container_id():
"""
Return the container ID of the LocalStack container
:return: container ID
"""
container_name = get_main_container_name()
try:
return DOCKER_CLIENT.get_container_id(container_name)
except ContainerException:
return None
def get_default_executor_mode() -> str:
"""
Returns the default docker executor mode, which is "docker" if the docker socket is available via the docker
client, or "local" otherwise.
:return:
"""
try:
return "docker" if DOCKER_CLIENT.has_docker() else "local"
except Exception:
return "local"
def get_important_image_hashes() -> Dict[str, str]:
result = {}
for image in DIAGNOSE_IMAGES:
try:
image_version = DOCKER_CLIENT.inspect_image(image, pull=False)["RepoDigests"]
except NoSuchImage:
image_version = "not_present"
except Exception as e:
image_version = f"error: {e}"
result[image] = image_version
return result
def update_images(image_list: List[str]):
from rich.markup import escape
from rich.progress import MofNCompleteColumn, Progress
from localstack.utils.container_utils.container_client import ContainerException
from localstack.utils.docker_utils import DOCKER_CLIENT
updated_count = 0
failed_count = 0
progress = Progress(*Progress.get_default_columns(),
MofNCompleteColumn(),
transient=True,
console=console)
with progress:
for image in progress.track(image_list,
description="Processing image..."):
try:
updated = False
hash_before_pull = DOCKER_CLIENT.inspect_image(
image_name=image, pull=False)["Id"]
DOCKER_CLIENT.pull_image(image)
if (hash_before_pull != DOCKER_CLIENT.inspect_image(
image_name=image, pull=False)["Id"]):
updated = True
updated_count += 1
console.print(
f":heavy_check_mark: Image {escape(image)} {'updated' if updated else 'up-to-date'}.",
style="bold" if updated else None,
highlight=False,
)
except ContainerException as e:
console.print(
f":heavy_multiplication_x: Image {escape(image)} pull failed: {e.message}",
style="bold red",
highlight=False,
)
failed_count += 1
console.rule()
console.print(
f"Images updated: {updated_count}, Images failed: {failed_count}, total images processed: {len(image_list)}."
)
def get_docker_image_details(image_name=None):
image_name = image_name or get_docker_image_to_start()
try:
result = DOCKER_CLIENT.inspect_image(image_name)
except ContainerException:
return {}
result = {
"id": result["Id"].replace("sha256:", "")[:12],
"tag": (result.get("RepoTags") or ["latest"])[0].split(":")[-1],
"created": result["Created"].split(".")[0],
}
return result
def get_main_container_name():
global MAIN_CONTAINER_NAME_CACHED
if MAIN_CONTAINER_NAME_CACHED is None:
hostname = os.environ.get("HOSTNAME")
if hostname:
try:
MAIN_CONTAINER_NAME_CACHED = DOCKER_CLIENT.get_container_name(hostname)
except ContainerException:
MAIN_CONTAINER_NAME_CACHED = config.MAIN_CONTAINER_NAME
else:
MAIN_CONTAINER_NAME_CACHED = config.MAIN_CONTAINER_NAME
return MAIN_CONTAINER_NAME_CACHED
def is_up(self) -> bool:
"""
Checks whether the container is running, and the Ready marker has been printed to the logs.
"""
if not self.is_container_running():
return False
logs = DOCKER_CLIENT.get_container_logs(self.container.name)
if constants.READY_MARKER_OUTPUT not in logs.splitlines():
return False
# also checks the edge port health status
return super().is_up()
def get_main_container_name():
"""
Returns the container name of the LocalStack container
:return: LocalStack container name
"""
hostname = os.environ.get("HOSTNAME")
if hostname:
try:
return DOCKER_CLIENT.get_container_name(hostname)
except ContainerException:
return config.MAIN_CONTAINER_NAME
else:
return config.MAIN_CONTAINER_NAME
def cmd_update_docker_images():
from localstack.utils.docker_utils import DOCKER_CLIENT
console.rule("Updating docker images")
all_images = DOCKER_CLIENT.get_docker_image_names(strip_latest=False)
image_prefixes = ["localstack/", "lambci/lambda:", "mlupin/docker-lambda:"]
localstack_images = [
image for image in all_images if any(
image.startswith(image_prefix)
or image.startswith(f"docker.io/{image_prefix}")
for image_prefix in image_prefixes)
]
update_images(localstack_images)
def cmd_ssh():
from localstack import config
from localstack.utils.docker_utils import DOCKER_CLIENT
from localstack.utils.run import run
if not DOCKER_CLIENT.is_container_running(config.MAIN_CONTAINER_NAME):
raise click.ClickException(
'Expected a running container named "%s", but found none' %
config.MAIN_CONTAINER_NAME)
try:
process = run("docker exec -it %s bash" % config.MAIN_CONTAINER_NAME,
tty=True)
process.wait()
except KeyboardInterrupt:
pass
def start(self, env_vars: Dict[str, str]) -> None:
self.executor_endpoint.start()
network = self.get_network_for_executor()
container_config = ContainerConfiguration(
image_name=self.get_image(),
name=self.id,
env_vars=env_vars,
network=network,
entrypoint=RAPID_ENTRYPOINT,
)
CONTAINER_CLIENT.create_container_from_config(container_config)
if not config.LAMBDA_PREBUILD_IMAGES:
CONTAINER_CLIENT.copy_into_container(
self.id, str(get_runtime_client_path()), RAPID_ENTRYPOINT)
CONTAINER_CLIENT.copy_into_container(
self.id,
f"{str(get_code_path_for_function(self.function_version))}/",
"/var/task/")
CONTAINER_CLIENT.start_container(self.id)
self.ip = CONTAINER_CLIENT.get_container_ipv4_for_network(
container_name_or_id=self.id, container_network=network)
self.executor_endpoint.container_address = self.ip
def prepare_docker_start():
# prepare environment for docker start
container_name = config.MAIN_CONTAINER_NAME
if DOCKER_CLIENT.is_container_running(container_name):
raise ContainerExists('LocalStack container named "%s" is already running' % container_name)
if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER:
print(
f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the "
f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})."
) # Logger is not initialized at this point, so the warning is displayed via print
os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1"
# make sure temp folder exists
mkdir(config.TMP_FOLDER)
try:
chmod_r(config.TMP_FOLDER, 0o777)
except Exception:
pass
def get_container_network_for_lambda():
global LAMBDA_CONTAINER_NETWORK
if config.LAMBDA_DOCKER_NETWORK:
return config.LAMBDA_DOCKER_NETWORK
if LAMBDA_CONTAINER_NETWORK is None:
try:
if config.is_in_docker:
networks = DOCKER_CLIENT.get_networks(
bootstrap.get_main_container_name())
LAMBDA_CONTAINER_NETWORK = networks[0]
else:
LAMBDA_CONTAINER_NETWORK = (
"bridge" # use the default bridge network in case of host mode
)
LOG.info("Determined lambda container network: %s",
LAMBDA_CONTAINER_NETWORK)
except Exception as e:
container_name = bootstrap.get_main_container_name()
LOG.info('Unable to get network name of main container "%s": %s',
container_name, e)
return LAMBDA_CONTAINER_NETWORK
def get_main_container_network() -> Optional[str]:
"""
Gets the main network of the LocalStack container (if we run in one, bridge otherwise)
If there are multiple networks connected to the LocalStack container, we choose the first as "main" network
:return: Network name
"""
main_container_network = None
try:
if config.is_in_docker:
networks = DOCKER_CLIENT.get_networks(get_main_container_name())
main_container_network = networks[0]
else:
main_container_network = "bridge" # use the default bridge network in case of host mode
LOG.info("Determined main container network: %s",
main_container_network)
except Exception as e:
container_name = get_main_container_name()
LOG.info('Unable to get network name of main container "%s": %s',
container_name, e)
return main_container_network
def wait_container_is_ready(timeout: Optional[float] = None):
"""Blocks until the localstack main container is running and the ready marker has been printed."""
container_name = config.MAIN_CONTAINER_NAME
started = time.time()
def is_container_running():
return DOCKER_CLIENT.is_container_running(container_name)
if not poll_condition(is_container_running, timeout=timeout):
return False
stream = DOCKER_CLIENT.stream_container_logs(container_name)
# create a timer that will terminate the log stream after the remaining timeout
timer = None
if timeout:
waited = time.time() - started
remaining = timeout - waited
# check the rare case that the timeout has already been reached
if remaining <= 0:
stream.close()
return False
timer = threading.Timer(remaining, stream.close)
timer.start()
try:
for line in stream:
line = line.decode("utf-8").strip()
if line == constants.READY_MARKER_OUTPUT:
return True
# EOF was reached or the stream was closed
return False
finally:
call_safe(stream.close)
if timer:
# make sure the timer is stopped (does nothing if it has already run)
timer.cancel()
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
# TODO: works only when running on the host, outside of Docker -> add a fallback if running in Docker?
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
docker_name = "tmp-ls-sfn"
DOCKER_CLIENT.run_container(
IMAGE_NAME_SFN_LOCAL,
remove=True,
entrypoint="",
name=docker_name,
detach=True,
command=["sleep", "15"],
)
time.sleep(5)
DOCKER_CLIENT.copy_from_container(
docker_name,
local_path=dirs.static_libs,
container_path="/home/stepfunctionslocal/")
path = Path(f"{dirs.static_libs}/stepfunctionslocal/")
for file in path.glob("*.jar"):
file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
rm_rf("%s/stepfunctionslocal" % dirs.static_libs)
# apply patches
for patch_class, patch_url in (
(SFN_PATCH_CLASS1, SFN_PATCH_CLASS_URL1),
(SFN_PATCH_CLASS2, SFN_PATCH_CLASS_URL2),
):
patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, patch_class)
if not os.path.exists(patch_class_file):
download(patch_url, patch_class_file)
cmd = 'cd "%s"; zip %s %s' % (
INSTALL_DIR_STEPFUNCTIONS,
INSTALL_PATH_STEPFUNCTIONS_JAR,
patch_class,
)
run(cmd)
def do_shutdown(self):
try:
DOCKER_CLIENT.stop_container(self.container.name)
except Exception as e:
LOG.info("error cleaning up localstack container %s: %s", self.container.name, e)
def is_container_running():
return DOCKER_CLIENT.is_container_running(container_name)
def is_container_running(self) -> bool:
return DOCKER_CLIENT.is_container_running(self.container.name)
def get_main_container_id():
container_name = get_main_container_name()
try:
return DOCKER_CLIENT.get_container_id(container_name)
except ContainerException:
return None
def get_main_container_ip():
container_name = get_main_container_name()
return DOCKER_CLIENT.get_container_ip(container_name)
def inspect_main_container() -> Union[str, Dict]:
try:
return DOCKER_CLIENT.inspect_container(get_main_container_name())
except Exception as e:
return f"inspect failed: {e}"
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
if not DOCKER_CLIENT.has_docker():
# TODO: works only when a docker socket is available -> add a fallback if running without Docker?
LOG.warning(
"Docker not available - skipping installation of StepFunctions dependency"
)
return
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
docker_name = "tmp-ls-sfn"
DOCKER_CLIENT.run_container(
IMAGE_NAME_SFN_LOCAL,
remove=True,
entrypoint="",
name=docker_name,
detach=True,
command=["sleep", "15"],
)
time.sleep(5)
DOCKER_CLIENT.copy_from_container(
docker_name,
local_path=dirs.static_libs,
container_path="/home/stepfunctionslocal/")
path = Path(f"{dirs.static_libs}/stepfunctionslocal/")
for file in path.glob("*.jar"):
file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
rm_rf(str(path))
classes = [
SFN_PATCH_CLASS1,
SFN_PATCH_CLASS2,
SFN_PATCH_CLASS_REGION,
SFN_PATCH_CLASS_STARTER,
SFN_PATCH_CLASS_ASYNC2SERVICEAPI,
SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED,
SFN_PATCH_FILE_METAINF,
]
for patch_class in classes:
patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}"
add_file_to_jar(patch_class,
patch_url,
target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR)
# special case for Manifest file - extract first, replace content, then update in JAR file
manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF",
"MANIFEST.MF")
if not os.path.exists(manifest_file):
content = run([
"unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR,
"META-INF/MANIFEST.MF"
])
content = re.sub("Main-Class: .+",
"Main-Class: cloud.localstack.StepFunctionsStarter",
content)
classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS])
content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ",
content)
save_file(manifest_file, content)
run(
["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"],
cwd=INSTALL_DIR_STEPFUNCTIONS,
)
# download additional jar libs
for jar_url in JAR_URLS:
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
os.path.basename(jar_url))
if not file_exists_not_empty(target):
download(jar_url, target)
# download aws-sdk lambda handler
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
"localstack-internal-awssdk", "awssdk.zip")
if not file_exists_not_empty(target):
download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def start_infra_in_docker():
container_name = config.MAIN_CONTAINER_NAME
if DOCKER_CLIENT.is_container_running(container_name):
raise Exception('LocalStack container named "%s" is already running' % container_name)
if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER:
print(
f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the "
f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})."
) # Logger is not initialized at this point, so the warning is displayed via print
os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1"
# load plugins before starting the docker container
plugin_configs = load_plugins()
# prepare APIs
canonicalize_api_names()
entrypoint = os.environ.get("ENTRYPOINT", "")
cmd = os.environ.get("CMD", "")
user_flags = config.DOCKER_FLAGS
image_name = get_docker_image_to_start()
service_ports = config.SERVICE_PORTS
force_noninteractive = os.environ.get("FORCE_NONINTERACTIVE", "")
# get run params
plugin_run_params = " ".join(
[entry.get("docker", {}).get("run_flags", "") for entry in plugin_configs]
)
# container for port mappings
port_mappings = PortMappings(bind_host=config.EDGE_BIND_HOST)
# get port ranges defined via DOCKER_FLAGS (if any)
user_flags = extract_port_flags(user_flags, port_mappings)
plugin_run_params = extract_port_flags(plugin_run_params, port_mappings)
# construct default port mappings
if service_ports.get("edge") == 0:
service_ports.pop("edge")
for port in service_ports.values():
port_mappings.add(port)
env_vars = {}
for env_var in config.CONFIG_ENV_VARS:
value = os.environ.get(env_var, None)
if value is not None:
env_vars[env_var] = value
bind_mounts = []
data_dir = os.environ.get("DATA_DIR", None)
if data_dir is not None:
container_data_dir = "/tmp/localstack_data"
bind_mounts.append((data_dir, container_data_dir))
env_vars["DATA_DIR"] = container_data_dir
bind_mounts.append((config.TMP_FOLDER, "/tmp/localstack"))
bind_mounts.append((config.DOCKER_SOCK, config.DOCKER_SOCK))
env_vars["DOCKER_HOST"] = f"unix://{config.DOCKER_SOCK}"
env_vars["HOST_TMP_FOLDER"] = config.HOST_TMP_FOLDER
if config.DEVELOP:
port_mappings.add(config.DEVELOP_PORT)
docker_cmd = [config.DOCKER_CMD, "run"]
if not force_noninteractive and not in_ci():
docker_cmd.append("-it")
if entrypoint:
docker_cmd += shlex.split(entrypoint)
if env_vars:
docker_cmd += [item for k, v in env_vars.items() for item in ["-e", "{}={}".format(k, v)]]
if user_flags:
docker_cmd += shlex.split(user_flags)
if plugin_run_params:
docker_cmd += shlex.split(plugin_run_params)
docker_cmd += ["--rm", "--privileged"]
docker_cmd += ["--name", container_name]
docker_cmd += port_mappings.to_list()
docker_cmd += [
volume
for host_path, docker_path in bind_mounts
for volume in ["-v", f"{host_path}:{docker_path}"]
]
docker_cmd.append(image_name)
docker_cmd += shlex.split(cmd)
mkdir(config.TMP_FOLDER)
try:
run(["chmod", "-R", "777", config.TMP_FOLDER], print_error=False, shell=False)
except Exception:
pass
class ShellRunnerThread(threading.Thread):
def __init__(self, cmd):
threading.Thread.__init__(self)
self.daemon = True
self.cmd = cmd
def run(self):
self.process = run(self.cmd, asynchronous=True, shell=False)
# keep this print output here for debugging purposes
print(docker_cmd)
t = ShellRunnerThread(docker_cmd)
t.start()
time.sleep(2)
if DO_CHMOD_DOCKER_SOCK:
# fix permissions on /var/run/docker.sock
for i in range(0, 100):
if DOCKER_CLIENT.is_container_running(container_name):
break
time.sleep(2)
DOCKER_CLIENT.exec_in_container(
container_name, command=["chmod", "777", "/var/run/docker.sock"], user="******"
)
t.process.wait()
sys.exit(t.process.returncode)
def _check_skip():
if not is_env_not_false("SKIP_DOCKER_TESTS"):
pytest.skip("SKIP_DOCKER_TESTS is set")
if not DOCKER_CLIENT.has_docker():
pytest.skip("Docker is not available")
