def test_argument_parsing():
test_port_string = "-p 80:8080/udp"
test_port_string_with_host = "-p 127.0.0.1:6000:7000/tcp"
test_env_string = "-e TEST_ENV_VAR=test_string=123"
test_mount_string = "-v /var/test:/opt/test"
argument_string = (
f"{test_port_string} {test_env_string} {test_mount_string} {test_port_string_with_host}"
)
env_vars = {}
ports = PortMappings()
mounts = []
Util.parse_additional_flags(argument_string, env_vars, ports, mounts)
assert env_vars == {"TEST_ENV_VAR": "test_string=123"}
assert ports.to_str() == "-p 80:8080/udp -p 6000:7000"
assert mounts == [("/var/test", "/opt/test")]
argument_string = (
"--add-host host.docker.internal:host-gateway --add-host arbitrary.host:127.0.0.1"
)
_, _, _, extra_hosts, _ = Util.parse_additional_flags(
argument_string, env_vars, ports, mounts)
assert {
"host.docker.internal": "host-gateway",
"arbitrary.host": "127.0.0.1"
} == extra_hosts
with pytest.raises(NotImplementedError):
argument_string = "--somerandomargument"
Util.parse_additional_flags(argument_string, env_vars, ports, mounts)
with pytest.raises(ValueError):
argument_string = "--publish 80:80:80:80"
Util.parse_additional_flags(argument_string, env_vars, ports, mounts)
# Test windows paths
argument_string = r'-v "C:\Users\SomeUser\SomePath:/var/task"'
_, _, mounts, _, _ = Util.parse_additional_flags(argument_string)
assert mounts == [(r"C:\Users\SomeUser\SomePath", "/var/task")]
argument_string = r'-v "C:\Users\SomeUser\SomePath:/var/task:ro"'
_, _, mounts, _, _ = Util.parse_additional_flags(argument_string)
assert mounts == [(r"C:\Users\SomeUser\SomePath", "/var/task")]
argument_string = r'-v "C:\Users\Some User\Some Path:/var/task:ro"'
_, _, mounts, _, _ = Util.parse_additional_flags(argument_string)
assert mounts == [(r"C:\Users\Some User\Some Path", "/var/task")]
argument_string = r'-v "/var/test:/var/task:ro"'
_, _, mounts, _, _ = Util.parse_additional_flags(argument_string)
assert mounts == [("/var/test", "/var/task")]
# Test file paths
argument_string = r'-v "/tmp/test.jar:/tmp/foo bar/test.jar"'
_, _, mounts, _, _ = Util.parse_additional_flags(argument_string)
assert mounts == [(r"/tmp/test.jar", "/tmp/foo bar/test.jar")]
argument_string = r'-v "/tmp/test-foo_bar.jar:/tmp/test-foo_bar2.jar"'
_, _, mounts, _, _ = Util.parse_additional_flags(argument_string)
assert mounts == [(r"/tmp/test-foo_bar.jar", "/tmp/test-foo_bar2.jar")]
# Test file paths
argument_string = r'-v "/tmp/test.jar:/tmp/foo bar/test.jar" --network mynet123'
_, _, _, _, network = Util.parse_additional_flags(argument_string)
assert network == "mynet123"
def extract_port_flags(user_flags, port_mappings: PortMappings):
regex = r"-p\s+([0-9]+)(\-([0-9]+))?:([0-9]+)(\-([0-9]+))?"
matches = re.match(".*%s" % regex, user_flags)
if matches:
for match in re.findall(regex, user_flags):
start = int(match[0])
end = int(match[2] or match[0])
start_target = int(match[3] or start)
end_target = int(match[5] or end)
port_mappings.add([start, end], [start_target, end_target])
user_flags = re.sub(regex, r"", user_flags)
return user_flags
def test_overlapping_port_ranges(self):
port_mappings = PortMappings()
port_mappings.add(4590)
port_mappings.add(4591)
port_mappings.add(4593)
port_mappings.add(4592)
port_mappings.add(4593)
result = port_mappings.to_str()
# assert that ranges are non-overlapping, i.e., no duplicate ports
self.assertEqual("-p 4590-4592:4590-4592 -p 4593:4593", result)
def test_port_ranges_with_bind_host(self):
port_mappings = PortMappings(bind_host="0.0.0.0")
port_mappings.add(5000)
port_mappings.add(5001)
port_mappings.add(5003)
port_mappings.add([5004, 5006], 9000)
result = port_mappings.to_str()
self.assertEqual(
"-p 0.0.0.0:5000-5001:5000-5001 -p 0.0.0.0:5003:5003 -p 0.0.0.0:5004-5006:9000",
result)
def test_port_ranges_with_bind_host_to_dict(self):
port_mappings = PortMappings(bind_host="0.0.0.0")
port_mappings.add(5000, 6000)
port_mappings.add(5001, 7000)
port_mappings.add(5003, 8000)
port_mappings.add([5004, 5006], 9000)
result = port_mappings.to_dict()
expected_result = {
"6000/tcp": ("0.0.0.0", 5000),
"7000/tcp": ("0.0.0.0", 5001),
"8000/tcp": ("0.0.0.0", 5003),
"9000/tcp": ("0.0.0.0", [5004, 5005, 5006]),
}
self.assertEqual(expected_result, result)
def __init__(self, name: str = None):
self.name = name or config.MAIN_CONTAINER_NAME
self.entrypoint = os.environ.get("ENTRYPOINT", "")
self.command = shlex.split(os.environ.get("CMD", ""))
self.image_name = get_docker_image_to_start()
self.ports = PortMappings(bind_host=config.EDGE_BIND_HOST)
self.volumes = VolumeMappings()
self.env_vars = dict()
self.additional_flags = list()
self.logfile = os.path.join(config.TMP_FOLDER, f"{self.name}_container.log")
def test_adjacent_port_to_many_to_one(self):
port_mappings = PortMappings()
port_mappings.add([7000, 7002], 7000)
port_mappings.add(6999)
expected_result = {
"6999/tcp": 6999,
"7000/tcp": [7000, 7001, 7002],
}
result = port_mappings.to_dict()
self.assertEqual(expected_result, result)
def test_many_to_one_adjacent_to_uniform(self):
port_mappings = PortMappings()
port_mappings.add(5002)
port_mappings.add(5003)
port_mappings.add([5004, 5006], 5004)
expected_result = {
"5002/tcp": 5002,
"5003/tcp": 5003,
"5004/tcp": [5004, 5005, 5006],
}
result = port_mappings.to_dict()
self.assertEqual(expected_result, result)
def test_extract_port_flags(self):
port_mappings = PortMappings()
flags = extract_port_flags("foo -p 1234:1234 bar",
port_mappings=port_mappings)
self.assertEqual("foo bar", flags)
mapping_str = port_mappings.to_str()
self.assertEqual("-p 1234:1234", mapping_str)
port_mappings = PortMappings()
flags = extract_port_flags("foo -p 1234:1234 bar -p 80-90:81-91 baz",
port_mappings=port_mappings)
self.assertEqual("foo bar baz", flags)
mapping_str = port_mappings.to_str()
self.assertIn("-p 1234:1234", mapping_str)
self.assertIn("-p 80-90:81-91", mapping_str)
def test_port_mappings(self):
map = PortMappings()
map.add(123)
self.assertEqual("-p 123:123", map.to_str())
map.add(124)
self.assertEqual("-p 123-124:123-124", map.to_str())
map.add(234)
self.assertEqual("-p 123-124:123-124 -p 234:234", map.to_str())
map.add(345, 346)
self.assertEqual("-p 123-124:123-124 -p 234:234 -p 345:346",
map.to_str())
map.add([456, 458])
self.assertEqual(
"-p 123-124:123-124 -p 234:234 -p 345:346 -p 456-458:456-458",
map.to_str())
map = PortMappings()
map.add([123, 124])
self.assertEqual("-p 123-124:123-124", map.to_str())
map.add([234, 237], [345, 348])
self.assertEqual("-p 123-124:123-124 -p 234-237:345-348", map.to_str())
def test_create_with_port_mapping(self, docker_client: ContainerClient,
create_container):
ports = PortMappings()
ports.add(45122, 22)
ports.add(45180, 80)
create_container("alpine", ports=ports)
def start_infra_in_docker():
container_name = config.MAIN_CONTAINER_NAME
if DOCKER_CLIENT.is_container_running(container_name):
raise Exception('LocalStack container named "%s" is already running' % container_name)
if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER:
print(
f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the "
f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})."
) # Logger is not initialized at this point, so the warning is displayed via print
os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1"
# load plugins before starting the docker container
plugin_configs = load_plugins()
# prepare APIs
canonicalize_api_names()
entrypoint = os.environ.get("ENTRYPOINT", "")
cmd = os.environ.get("CMD", "")
user_flags = config.DOCKER_FLAGS
image_name = get_docker_image_to_start()
service_ports = config.SERVICE_PORTS
force_noninteractive = os.environ.get("FORCE_NONINTERACTIVE", "")
# get run params
plugin_run_params = " ".join(
[entry.get("docker", {}).get("run_flags", "") for entry in plugin_configs]
)
# container for port mappings
port_mappings = PortMappings(bind_host=config.EDGE_BIND_HOST)
# get port ranges defined via DOCKER_FLAGS (if any)
user_flags = extract_port_flags(user_flags, port_mappings)
plugin_run_params = extract_port_flags(plugin_run_params, port_mappings)
# construct default port mappings
if service_ports.get("edge") == 0:
service_ports.pop("edge")
for port in service_ports.values():
port_mappings.add(port)
env_vars = {}
for env_var in config.CONFIG_ENV_VARS:
value = os.environ.get(env_var, None)
if value is not None:
env_vars[env_var] = value
bind_mounts = []
data_dir = os.environ.get("DATA_DIR", None)
if data_dir is not None:
container_data_dir = "/tmp/localstack_data"
bind_mounts.append((data_dir, container_data_dir))
env_vars["DATA_DIR"] = container_data_dir
bind_mounts.append((config.TMP_FOLDER, "/tmp/localstack"))
bind_mounts.append((config.DOCKER_SOCK, config.DOCKER_SOCK))
env_vars["DOCKER_HOST"] = f"unix://{config.DOCKER_SOCK}"
env_vars["HOST_TMP_FOLDER"] = config.HOST_TMP_FOLDER
if config.DEVELOP:
port_mappings.add(config.DEVELOP_PORT)
docker_cmd = [config.DOCKER_CMD, "run"]
if not force_noninteractive and not in_ci():
docker_cmd.append("-it")
if entrypoint:
docker_cmd += shlex.split(entrypoint)
if env_vars:
docker_cmd += [item for k, v in env_vars.items() for item in ["-e", "{}={}".format(k, v)]]
if user_flags:
docker_cmd += shlex.split(user_flags)
if plugin_run_params:
docker_cmd += shlex.split(plugin_run_params)
docker_cmd += ["--rm", "--privileged"]
docker_cmd += ["--name", container_name]
docker_cmd += port_mappings.to_list()
docker_cmd += [
volume
for host_path, docker_path in bind_mounts
for volume in ["-v", f"{host_path}:{docker_path}"]
]
docker_cmd.append(image_name)
docker_cmd += shlex.split(cmd)
mkdir(config.TMP_FOLDER)
try:
run(["chmod", "-R", "777", config.TMP_FOLDER], print_error=False, shell=False)
except Exception:
pass
class ShellRunnerThread(threading.Thread):
def __init__(self, cmd):
threading.Thread.__init__(self)
self.daemon = True
self.cmd = cmd
def run(self):
self.process = run(self.cmd, asynchronous=True, shell=False)
# keep this print output here for debugging purposes
print(docker_cmd)
t = ShellRunnerThread(docker_cmd)
t.start()
time.sleep(2)
if DO_CHMOD_DOCKER_SOCK:
# fix permissions on /var/run/docker.sock
for i in range(0, 100):
if DOCKER_CLIENT.is_container_running(container_name):
break
time.sleep(2)
DOCKER_CLIENT.exec_in_container(
container_name, command=["chmod", "777", "/var/run/docker.sock"], user="******"
)
t.process.wait()
sys.exit(t.process.returncode)