def run(self):
self.process = run(self.cmd, asynchronous=True, shell=False)
def start_infra_in_docker():
container_name = config.MAIN_CONTAINER_NAME
if DOCKER_CLIENT.is_container_running(container_name):
raise Exception('LocalStack container named "%s" is already running' %
container_name)
if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER:
print(
f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the "
f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})."
) # Logger is not initialized at this point, so the warning is displayed via print
os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1"
# load plugins before starting the docker container
plugin_configs = load_plugins()
# prepare APIs
canonicalize_api_names()
entrypoint = os.environ.get("ENTRYPOINT", "")
cmd = os.environ.get("CMD", "")
user_flags = config.DOCKER_FLAGS
image_name = get_docker_image_to_start()
service_ports = config.SERVICE_PORTS
force_noninteractive = os.environ.get("FORCE_NONINTERACTIVE", "")
# get run params
plugin_run_params = " ".join([
entry.get("docker", {}).get("run_flags", "")
for entry in plugin_configs
])
# container for port mappings
port_mappings = PortMappings(bind_host=config.EDGE_BIND_HOST)
# get port ranges defined via DOCKER_FLAGS (if any)
user_flags = extract_port_flags(user_flags, port_mappings)
plugin_run_params = extract_port_flags(plugin_run_params, port_mappings)
# construct default port mappings
if service_ports.get("edge") == 0:
service_ports.pop("edge")
for port in service_ports.values():
if port:
port_mappings.add(port)
env_vars = {}
for env_var in config.CONFIG_ENV_VARS:
value = os.environ.get(env_var, None)
if value is not None:
env_vars[env_var] = value
bind_mounts = []
data_dir = os.environ.get("DATA_DIR", None)
if data_dir is not None:
container_data_dir = "/tmp/localstack_data"
bind_mounts.append((data_dir, container_data_dir))
env_vars["DATA_DIR"] = container_data_dir
bind_mounts.append((config.TMP_FOLDER, "/tmp/localstack"))
bind_mounts.append((config.DOCKER_SOCK, config.DOCKER_SOCK))
env_vars["DOCKER_HOST"] = f"unix://{config.DOCKER_SOCK}"
env_vars["HOST_TMP_FOLDER"] = config.HOST_TMP_FOLDER
if config.DEVELOP:
port_mappings.add(config.DEVELOP_PORT)
docker_cmd = [config.DOCKER_CMD, "run"]
if not force_noninteractive and not in_ci():
docker_cmd.append("-it")
if entrypoint:
docker_cmd += shlex.split(entrypoint)
if env_vars:
docker_cmd += [
item for k, v in env_vars.items()
for item in ["-e", "{}={}".format(k, v)]
]
if user_flags:
docker_cmd += shlex.split(user_flags)
if plugin_run_params:
docker_cmd += shlex.split(plugin_run_params)
docker_cmd += ["--rm", "--privileged"]
docker_cmd += ["--name", container_name]
docker_cmd += port_mappings.to_list()
docker_cmd += [
volume for host_path, docker_path in bind_mounts
for volume in ["-v", f"{host_path}:{docker_path}"]
]
docker_cmd.append(image_name)
docker_cmd += shlex.split(cmd)
mkdir(config.TMP_FOLDER)
try:
run(["chmod", "-R", "777", config.TMP_FOLDER],
print_error=False,
shell=False)
except Exception:
pass
class ShellRunnerThread(threading.Thread):
def __init__(self, cmd):
threading.Thread.__init__(self)
self.daemon = True
self.cmd = cmd
def run(self):
self.process = run(self.cmd, asynchronous=True, shell=False)
# keep this print output here for debugging purposes
print(docker_cmd)
t = ShellRunnerThread(docker_cmd)
t.start()
time.sleep(2)
if DO_CHMOD_DOCKER_SOCK:
# fix permissions on /var/run/docker.sock
for i in range(0, 100):
if DOCKER_CLIENT.is_container_running(container_name):
break
time.sleep(2)
DOCKER_CLIENT.exec_in_container(
container_name,
command=["chmod", "777", "/var/run/docker.sock"],
user="******")
t.process.wait()
sys.exit(t.process.returncode)
def validate_localstack_config(name):
# TODO: separate functionality from CLI output
# (use exceptions to communicate errors, and return list of warnings)
from subprocess import CalledProcessError
from localstack.cli import console
dirname = os.getcwd()
compose_file_name = name if os.path.isabs(name) else os.path.join(
dirname, name)
warns = []
# validating docker-compose file
cmd = ["docker-compose", "-f", compose_file_name, "config"]
try:
run(cmd, shell=False, print_error=False)
except CalledProcessError as e:
msg = f"{e}\n{to_str(e.output)}".strip()
raise ValueError(msg)
# validating docker-compose variable
import yaml # keep import here to avoid issues in test Lambdas
with open(compose_file_name) as file:
compose_content = yaml.full_load(file)
services_config = compose_content.get("services", {})
ls_service_name = [
name for name, svc in services_config.items()
if "localstack" in svc.get("image", "")
]
if not ls_service_name:
raise Exception(
'No LocalStack service found in config (looking for image names containing "localstack")'
)
if len(ls_service_name) > 1:
warns.append(
f"Multiple candidates found for LocalStack service: {ls_service_name}"
)
ls_service_name = ls_service_name[0]
ls_service_details = services_config[ls_service_name]
image_name = ls_service_details.get("image", "")
if image_name.split(":")[0] not in constants.OFFICIAL_IMAGES:
warns.append(
'Using custom image "%s", we recommend using an official image: %s'
% (image_name, constants.OFFICIAL_IMAGES))
# prepare config options
network_mode = ls_service_details.get("network_mode")
image_name = ls_service_details.get("image")
container_name = ls_service_details.get("container_name") or ""
docker_ports = (port.split(":")[-2]
for port in ls_service_details.get("ports", []))
docker_env = dict((env.split("=")[0], env.split("=")[1])
for env in ls_service_details.get("environment", {}))
edge_port = str(docker_env.get("EDGE_PORT") or config.EDGE_PORT)
main_container = config.MAIN_CONTAINER_NAME
# docker-compose file validation cases
if (docker_env.get("PORT_WEB_UI") not in ["${PORT_WEB_UI- }", None, ""]
and image_name == "localstack/localstack"):
warns.append(
'"PORT_WEB_UI" Web UI is now deprecated, '
'and requires to use the "localstack/localstack-full" image.')
if not docker_env.get("HOST_TMP_FOLDER"):
warns.append(
'Please configure the "HOST_TMP_FOLDER" environment variable to point to the '
+
"absolute path of a temp folder on your host system (e.g., HOST_TMP_FOLDER=${TMPDIR})"
)
if (main_container not in container_name
) and not docker_env.get("MAIN_CONTAINER_NAME"):
warns.append(
'Please use "container_name: %s" or add "MAIN_CONTAINER_NAME" in "environment".'
% main_container)
def port_exposed(port):
for exposed in docker_ports:
if re.match(r"^([0-9]+-)?%s(-[0-9]+)?$" % port, exposed):
return True
if not port_exposed(edge_port):
warns.append(
("Edge port %s is not exposed. You may have to add the entry "
'to the "ports" section of the docker-compose file.') % edge_port)
if network_mode != "bridge" and not docker_env.get(
"LAMBDA_DOCKER_NETWORK"):
warns.append(
'Network mode is not set to "bridge" which may cause networking issues in Lambda containers. '
'Consider adding "network_mode: bridge" to your docker-compose file, or configure '
"LAMBDA_DOCKER_NETWORK with the name of the Docker network of your compose stack."
)
# print warning/info messages
for warning in warns:
console.print("[yellow]:warning:[/yellow]", warning)
if not warns:
return True
return False
def add_file_to_jar(class_file, class_url, target_jar, base_dir=None):
base_dir = base_dir or os.path.dirname(target_jar)
patch_class_file = os.path.join(base_dir, class_file)
if not os.path.exists(patch_class_file):
download(class_url, patch_class_file)
run(["zip", target_jar, class_file], cwd=base_dir)
def install_stepfunctions_local():
if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR):
# pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file
if not DOCKER_CLIENT.has_docker():
# TODO: works only when a docker socket is available -> add a fallback if running without Docker?
LOG.warning(
"Docker not available - skipping installation of StepFunctions dependency"
)
return
log_install_msg("Step Functions")
mkdir(INSTALL_DIR_STEPFUNCTIONS)
DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL)
docker_name = "tmp-ls-sfn"
DOCKER_CLIENT.run_container(
IMAGE_NAME_SFN_LOCAL,
remove=True,
entrypoint="",
name=docker_name,
detach=True,
command=["sleep", "15"],
)
time.sleep(5)
DOCKER_CLIENT.copy_from_container(
docker_name,
local_path=dirs.static_libs,
container_path="/home/stepfunctionslocal/")
path = Path(f"{dirs.static_libs}/stepfunctionslocal/")
for file in path.glob("*.jar"):
file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name)
rm_rf(str(path))
classes = [
SFN_PATCH_CLASS1,
SFN_PATCH_CLASS2,
SFN_PATCH_CLASS_REGION,
SFN_PATCH_CLASS_STARTER,
SFN_PATCH_CLASS_ASYNC2SERVICEAPI,
SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED,
SFN_PATCH_FILE_METAINF,
]
for patch_class in classes:
patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}"
add_file_to_jar(patch_class,
patch_url,
target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR)
# special case for Manifest file - extract first, replace content, then update in JAR file
manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF",
"MANIFEST.MF")
if not os.path.exists(manifest_file):
content = run([
"unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR,
"META-INF/MANIFEST.MF"
])
content = re.sub("Main-Class: .+",
"Main-Class: cloud.localstack.StepFunctionsStarter",
content)
classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS])
content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ",
content)
save_file(manifest_file, content)
run(
["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"],
cwd=INSTALL_DIR_STEPFUNCTIONS,
)
# download additional jar libs
for jar_url in JAR_URLS:
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
os.path.basename(jar_url))
if not file_exists_not_empty(target):
download(jar_url, target)
# download aws-sdk lambda handler
target = os.path.join(INSTALL_DIR_STEPFUNCTIONS,
"localstack-internal-awssdk", "awssdk.zip")
if not file_exists_not_empty(target):
download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def try_install():
output = run([plugin_binary, "install", "-b", plugin])
LOG.debug("Plugin installation output: %s", output)