def get_image_environment_variable(env_name: str) -> Optional[str]: image_name = get_docker_image_to_start() image_info = DOCKER_CLIENT.inspect_image(image_name) image_envs = image_info["Config"]["Env"] try: found_env = next(env for env in image_envs if env.startswith(env_name)) except StopIteration: return None return found_env.split("=")[1]
def get_main_container_id(): """ Return the container ID of the LocalStack container :return: container ID """ container_name = get_main_container_name() try: return DOCKER_CLIENT.get_container_id(container_name) except ContainerException: return None
def get_default_executor_mode() -> str: """ Returns the default docker executor mode, which is "docker" if the docker socket is available via the docker client, or "local" otherwise. :return: """ try: return "docker" if DOCKER_CLIENT.has_docker() else "local" except Exception: return "local"
def get_important_image_hashes() -> Dict[str, str]: result = {} for image in DIAGNOSE_IMAGES: try: image_version = DOCKER_CLIENT.inspect_image(image, pull=False)["RepoDigests"] except NoSuchImage: image_version = "not_present" except Exception as e: image_version = f"error: {e}" result[image] = image_version return result
def update_images(image_list: List[str]): from rich.markup import escape from rich.progress import MofNCompleteColumn, Progress from localstack.utils.container_utils.container_client import ContainerException from localstack.utils.docker_utils import DOCKER_CLIENT updated_count = 0 failed_count = 0 progress = Progress(*Progress.get_default_columns(), MofNCompleteColumn(), transient=True, console=console) with progress: for image in progress.track(image_list, description="Processing image..."): try: updated = False hash_before_pull = DOCKER_CLIENT.inspect_image( image_name=image, pull=False)["Id"] DOCKER_CLIENT.pull_image(image) if (hash_before_pull != DOCKER_CLIENT.inspect_image( image_name=image, pull=False)["Id"]): updated = True updated_count += 1 console.print( f":heavy_check_mark: Image {escape(image)} {'updated' if updated else 'up-to-date'}.", style="bold" if updated else None, highlight=False, ) except ContainerException as e: console.print( f":heavy_multiplication_x: Image {escape(image)} pull failed: {e.message}", style="bold red", highlight=False, ) failed_count += 1 console.rule() console.print( f"Images updated: {updated_count}, Images failed: {failed_count}, total images processed: {len(image_list)}." )
def get_docker_image_details(image_name=None): image_name = image_name or get_docker_image_to_start() try: result = DOCKER_CLIENT.inspect_image(image_name) except ContainerException: return {} result = { "id": result["Id"].replace("sha256:", "")[:12], "tag": (result.get("RepoTags") or ["latest"])[0].split(":")[-1], "created": result["Created"].split(".")[0], } return result
def get_main_container_name(): global MAIN_CONTAINER_NAME_CACHED if MAIN_CONTAINER_NAME_CACHED is None: hostname = os.environ.get("HOSTNAME") if hostname: try: MAIN_CONTAINER_NAME_CACHED = DOCKER_CLIENT.get_container_name(hostname) except ContainerException: MAIN_CONTAINER_NAME_CACHED = config.MAIN_CONTAINER_NAME else: MAIN_CONTAINER_NAME_CACHED = config.MAIN_CONTAINER_NAME return MAIN_CONTAINER_NAME_CACHED
def is_up(self) -> bool: """ Checks whether the container is running, and the Ready marker has been printed to the logs. """ if not self.is_container_running(): return False logs = DOCKER_CLIENT.get_container_logs(self.container.name) if constants.READY_MARKER_OUTPUT not in logs.splitlines(): return False # also checks the edge port health status return super().is_up()
def get_main_container_name(): """ Returns the container name of the LocalStack container :return: LocalStack container name """ hostname = os.environ.get("HOSTNAME") if hostname: try: return DOCKER_CLIENT.get_container_name(hostname) except ContainerException: return config.MAIN_CONTAINER_NAME else: return config.MAIN_CONTAINER_NAME
def cmd_update_docker_images(): from localstack.utils.docker_utils import DOCKER_CLIENT console.rule("Updating docker images") all_images = DOCKER_CLIENT.get_docker_image_names(strip_latest=False) image_prefixes = ["localstack/", "lambci/lambda:", "mlupin/docker-lambda:"] localstack_images = [ image for image in all_images if any( image.startswith(image_prefix) or image.startswith(f"docker.io/{image_prefix}") for image_prefix in image_prefixes) ] update_images(localstack_images)
def cmd_ssh(): from localstack import config from localstack.utils.docker_utils import DOCKER_CLIENT from localstack.utils.run import run if not DOCKER_CLIENT.is_container_running(config.MAIN_CONTAINER_NAME): raise click.ClickException( 'Expected a running container named "%s", but found none' % config.MAIN_CONTAINER_NAME) try: process = run("docker exec -it %s bash" % config.MAIN_CONTAINER_NAME, tty=True) process.wait() except KeyboardInterrupt: pass
def start(self, env_vars: Dict[str, str]) -> None: self.executor_endpoint.start() network = self.get_network_for_executor() container_config = ContainerConfiguration( image_name=self.get_image(), name=self.id, env_vars=env_vars, network=network, entrypoint=RAPID_ENTRYPOINT, ) CONTAINER_CLIENT.create_container_from_config(container_config) if not config.LAMBDA_PREBUILD_IMAGES: CONTAINER_CLIENT.copy_into_container( self.id, str(get_runtime_client_path()), RAPID_ENTRYPOINT) CONTAINER_CLIENT.copy_into_container( self.id, f"{str(get_code_path_for_function(self.function_version))}/", "/var/task/") CONTAINER_CLIENT.start_container(self.id) self.ip = CONTAINER_CLIENT.get_container_ipv4_for_network( container_name_or_id=self.id, container_network=network) self.executor_endpoint.container_address = self.ip
def prepare_docker_start(): # prepare environment for docker start container_name = config.MAIN_CONTAINER_NAME if DOCKER_CLIENT.is_container_running(container_name): raise ContainerExists('LocalStack container named "%s" is already running' % container_name) if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER: print( f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the " f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})." ) # Logger is not initialized at this point, so the warning is displayed via print os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1" # make sure temp folder exists mkdir(config.TMP_FOLDER) try: chmod_r(config.TMP_FOLDER, 0o777) except Exception: pass
def get_container_network_for_lambda(): global LAMBDA_CONTAINER_NETWORK if config.LAMBDA_DOCKER_NETWORK: return config.LAMBDA_DOCKER_NETWORK if LAMBDA_CONTAINER_NETWORK is None: try: if config.is_in_docker: networks = DOCKER_CLIENT.get_networks( bootstrap.get_main_container_name()) LAMBDA_CONTAINER_NETWORK = networks[0] else: LAMBDA_CONTAINER_NETWORK = ( "bridge" # use the default bridge network in case of host mode ) LOG.info("Determined lambda container network: %s", LAMBDA_CONTAINER_NETWORK) except Exception as e: container_name = bootstrap.get_main_container_name() LOG.info('Unable to get network name of main container "%s": %s', container_name, e) return LAMBDA_CONTAINER_NETWORK
def get_main_container_network() -> Optional[str]: """ Gets the main network of the LocalStack container (if we run in one, bridge otherwise) If there are multiple networks connected to the LocalStack container, we choose the first as "main" network :return: Network name """ main_container_network = None try: if config.is_in_docker: networks = DOCKER_CLIENT.get_networks(get_main_container_name()) main_container_network = networks[0] else: main_container_network = "bridge" # use the default bridge network in case of host mode LOG.info("Determined main container network: %s", main_container_network) except Exception as e: container_name = get_main_container_name() LOG.info('Unable to get network name of main container "%s": %s', container_name, e) return main_container_network
def wait_container_is_ready(timeout: Optional[float] = None): """Blocks until the localstack main container is running and the ready marker has been printed.""" container_name = config.MAIN_CONTAINER_NAME started = time.time() def is_container_running(): return DOCKER_CLIENT.is_container_running(container_name) if not poll_condition(is_container_running, timeout=timeout): return False stream = DOCKER_CLIENT.stream_container_logs(container_name) # create a timer that will terminate the log stream after the remaining timeout timer = None if timeout: waited = time.time() - started remaining = timeout - waited # check the rare case that the timeout has already been reached if remaining <= 0: stream.close() return False timer = threading.Timer(remaining, stream.close) timer.start() try: for line in stream: line = line.decode("utf-8").strip() if line == constants.READY_MARKER_OUTPUT: return True # EOF was reached or the stream was closed return False finally: call_safe(stream.close) if timer: # make sure the timer is stopped (does nothing if it has already run) timer.cancel()
def install_stepfunctions_local(): if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR): # pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file # TODO: works only when running on the host, outside of Docker -> add a fallback if running in Docker? log_install_msg("Step Functions") mkdir(INSTALL_DIR_STEPFUNCTIONS) DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL) docker_name = "tmp-ls-sfn" DOCKER_CLIENT.run_container( IMAGE_NAME_SFN_LOCAL, remove=True, entrypoint="", name=docker_name, detach=True, command=["sleep", "15"], ) time.sleep(5) DOCKER_CLIENT.copy_from_container( docker_name, local_path=dirs.static_libs, container_path="/home/stepfunctionslocal/") path = Path(f"{dirs.static_libs}/stepfunctionslocal/") for file in path.glob("*.jar"): file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name) rm_rf("%s/stepfunctionslocal" % dirs.static_libs) # apply patches for patch_class, patch_url in ( (SFN_PATCH_CLASS1, SFN_PATCH_CLASS_URL1), (SFN_PATCH_CLASS2, SFN_PATCH_CLASS_URL2), ): patch_class_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, patch_class) if not os.path.exists(patch_class_file): download(patch_url, patch_class_file) cmd = 'cd "%s"; zip %s %s' % ( INSTALL_DIR_STEPFUNCTIONS, INSTALL_PATH_STEPFUNCTIONS_JAR, patch_class, ) run(cmd)
def do_shutdown(self): try: DOCKER_CLIENT.stop_container(self.container.name) except Exception as e: LOG.info("error cleaning up localstack container %s: %s", self.container.name, e)
def is_container_running(): return DOCKER_CLIENT.is_container_running(container_name)
def is_container_running(self) -> bool: return DOCKER_CLIENT.is_container_running(self.container.name)
def get_main_container_id(): container_name = get_main_container_name() try: return DOCKER_CLIENT.get_container_id(container_name) except ContainerException: return None
def get_main_container_ip(): container_name = get_main_container_name() return DOCKER_CLIENT.get_container_ip(container_name)
def inspect_main_container() -> Union[str, Dict]: try: return DOCKER_CLIENT.inspect_container(get_main_container_name()) except Exception as e: return f"inspect failed: {e}"
def install_stepfunctions_local(): if not os.path.exists(INSTALL_PATH_STEPFUNCTIONS_JAR): # pull the JAR file from the Docker image, which is more up-to-date than the downloadable JAR file if not DOCKER_CLIENT.has_docker(): # TODO: works only when a docker socket is available -> add a fallback if running without Docker? LOG.warning( "Docker not available - skipping installation of StepFunctions dependency" ) return log_install_msg("Step Functions") mkdir(INSTALL_DIR_STEPFUNCTIONS) DOCKER_CLIENT.pull_image(IMAGE_NAME_SFN_LOCAL) docker_name = "tmp-ls-sfn" DOCKER_CLIENT.run_container( IMAGE_NAME_SFN_LOCAL, remove=True, entrypoint="", name=docker_name, detach=True, command=["sleep", "15"], ) time.sleep(5) DOCKER_CLIENT.copy_from_container( docker_name, local_path=dirs.static_libs, container_path="/home/stepfunctionslocal/") path = Path(f"{dirs.static_libs}/stepfunctionslocal/") for file in path.glob("*.jar"): file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name) rm_rf(str(path)) classes = [ SFN_PATCH_CLASS1, SFN_PATCH_CLASS2, SFN_PATCH_CLASS_REGION, SFN_PATCH_CLASS_STARTER, SFN_PATCH_CLASS_ASYNC2SERVICEAPI, SFN_PATCH_CLASS_DESCRIBEEXECUTIONPARSED, SFN_PATCH_FILE_METAINF, ] for patch_class in classes: patch_url = f"{SFN_PATCH_URL_PREFIX}/{patch_class}" add_file_to_jar(patch_class, patch_url, target_jar=INSTALL_PATH_STEPFUNCTIONS_JAR) # special case for Manifest file - extract first, replace content, then update in JAR file manifest_file = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "META-INF", "MANIFEST.MF") if not os.path.exists(manifest_file): content = run([ "unzip", "-p", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF" ]) content = re.sub("Main-Class: .+", "Main-Class: cloud.localstack.StepFunctionsStarter", content) classpath = " ".join([os.path.basename(jar) for jar in JAR_URLS]) content = re.sub(r"Class-Path: \. ", f"Class-Path: {classpath} . ", content) save_file(manifest_file, content) run( ["zip", INSTALL_PATH_STEPFUNCTIONS_JAR, "META-INF/MANIFEST.MF"], cwd=INSTALL_DIR_STEPFUNCTIONS, ) # download additional jar libs for jar_url in JAR_URLS: target = os.path.join(INSTALL_DIR_STEPFUNCTIONS, os.path.basename(jar_url)) if not file_exists_not_empty(target): download(jar_url, target) # download aws-sdk lambda handler target = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "localstack-internal-awssdk", "awssdk.zip") if not file_exists_not_empty(target): download(SFN_AWS_SDK_LAMBDA_ZIP_FILE, target)
def start_infra_in_docker(): container_name = config.MAIN_CONTAINER_NAME if DOCKER_CLIENT.is_container_running(container_name): raise Exception('LocalStack container named "%s" is already running' % container_name) if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER: print( f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the " f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})." ) # Logger is not initialized at this point, so the warning is displayed via print os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1" # load plugins before starting the docker container plugin_configs = load_plugins() # prepare APIs canonicalize_api_names() entrypoint = os.environ.get("ENTRYPOINT", "") cmd = os.environ.get("CMD", "") user_flags = config.DOCKER_FLAGS image_name = get_docker_image_to_start() service_ports = config.SERVICE_PORTS force_noninteractive = os.environ.get("FORCE_NONINTERACTIVE", "") # get run params plugin_run_params = " ".join( [entry.get("docker", {}).get("run_flags", "") for entry in plugin_configs] ) # container for port mappings port_mappings = PortMappings(bind_host=config.EDGE_BIND_HOST) # get port ranges defined via DOCKER_FLAGS (if any) user_flags = extract_port_flags(user_flags, port_mappings) plugin_run_params = extract_port_flags(plugin_run_params, port_mappings) # construct default port mappings if service_ports.get("edge") == 0: service_ports.pop("edge") for port in service_ports.values(): port_mappings.add(port) env_vars = {} for env_var in config.CONFIG_ENV_VARS: value = os.environ.get(env_var, None) if value is not None: env_vars[env_var] = value bind_mounts = [] data_dir = os.environ.get("DATA_DIR", None) if data_dir is not None: container_data_dir = "/tmp/localstack_data" bind_mounts.append((data_dir, container_data_dir)) env_vars["DATA_DIR"] = container_data_dir bind_mounts.append((config.TMP_FOLDER, "/tmp/localstack")) bind_mounts.append((config.DOCKER_SOCK, config.DOCKER_SOCK)) env_vars["DOCKER_HOST"] = f"unix://{config.DOCKER_SOCK}" env_vars["HOST_TMP_FOLDER"] = config.HOST_TMP_FOLDER if config.DEVELOP: port_mappings.add(config.DEVELOP_PORT) docker_cmd = [config.DOCKER_CMD, "run"] if not force_noninteractive and not in_ci(): docker_cmd.append("-it") if entrypoint: docker_cmd += shlex.split(entrypoint) if env_vars: docker_cmd += [item for k, v in env_vars.items() for item in ["-e", "{}={}".format(k, v)]] if user_flags: docker_cmd += shlex.split(user_flags) if plugin_run_params: docker_cmd += shlex.split(plugin_run_params) docker_cmd += ["--rm", "--privileged"] docker_cmd += ["--name", container_name] docker_cmd += port_mappings.to_list() docker_cmd += [ volume for host_path, docker_path in bind_mounts for volume in ["-v", f"{host_path}:{docker_path}"] ] docker_cmd.append(image_name) docker_cmd += shlex.split(cmd) mkdir(config.TMP_FOLDER) try: run(["chmod", "-R", "777", config.TMP_FOLDER], print_error=False, shell=False) except Exception: pass class ShellRunnerThread(threading.Thread): def __init__(self, cmd): threading.Thread.__init__(self) self.daemon = True self.cmd = cmd def run(self): self.process = run(self.cmd, asynchronous=True, shell=False) # keep this print output here for debugging purposes print(docker_cmd) t = ShellRunnerThread(docker_cmd) t.start() time.sleep(2) if DO_CHMOD_DOCKER_SOCK: # fix permissions on /var/run/docker.sock for i in range(0, 100): if DOCKER_CLIENT.is_container_running(container_name): break time.sleep(2) DOCKER_CLIENT.exec_in_container( container_name, command=["chmod", "777", "/var/run/docker.sock"], user="******" ) t.process.wait() sys.exit(t.process.returncode)
def _check_skip(): if not is_env_not_false("SKIP_DOCKER_TESTS"): pytest.skip("SKIP_DOCKER_TESTS is set") if not DOCKER_CLIENT.has_docker(): pytest.skip("Docker is not available")