def docker_start_cmds(user, image, mount_dict, container_name, user_options,
cluster_name, home_directory, docker_cmd):
# Imported here due to circular dependency.
from ray.autoscaler.sdk import get_docker_host_mount_location
docker_mount_prefix = get_docker_host_mount_location(cluster_name)
mount = {f"{docker_mount_prefix}/{dst}": dst for dst in mount_dict}
mount_flags = " ".join([
"-v {src}:{dest}".format(src=k,
dest=v.replace("~/", home_directory + "/"))
for k, v in mount.items()
])
# for click, used in ray cli
env_vars = {"LC_ALL": "C.UTF-8", "LANG": "C.UTF-8"}
env_flags = " ".join(
["-e {name}={val}".format(name=k, val=v) for k, v in env_vars.items()])
user_options_str = " ".join(user_options)
docker_run = [
docker_cmd, "run", "--rm", "--name {}".format(container_name), "-d",
"-it", mount_flags, env_flags, user_options_str, "--net=host", image,
"bash"
]
return " ".join(docker_run)
def _get_docker_host_mount_location(self, cluster_name: str) -> str:
"""Return the docker host mount directory location."""
# Imported here due to circular dependency in imports.
from ray.autoscaler.sdk import get_docker_host_mount_location
return get_docker_host_mount_location(cluster_name)
def test_docker_rsync():
process_runner = MockProcessRunner()
provider = MockProvider()
provider.create_node({}, {}, 1)
cluster_name = "cluster"
docker_config = {"container_name": "container"}
args = {
"log_prefix": "prefix",
"node_id": 0,
"provider": provider,
"auth_config": auth_config,
"cluster_name": cluster_name,
"process_runner": process_runner,
"use_internal_ip": False,
"docker_config": docker_config,
}
cmd_runner = DockerCommandRunner(**args)
local_mount = "/home/ubuntu/base/mount/"
remote_mount = "/root/protected_mount/"
docker_mount_prefix = get_docker_host_mount_location(cluster_name)
remote_host_mount = f"{docker_mount_prefix}{remote_mount}"
local_file = "/home/ubuntu/base-file"
remote_file = "/root/protected-file"
remote_host_file = f"{docker_mount_prefix}{remote_file}"
process_runner.respond_to_call("docker inspect -f", ["true"])
cmd_runner.run_rsync_up(local_mount,
remote_mount,
options={"docker_mount_if_possible": True})
# Make sure we do not copy directly to raw destination
process_runner.assert_not_has_call(
"1.2.3.4", pattern=f"-avz {local_mount} [email protected]:{remote_mount}")
process_runner.assert_not_has_call("1.2.3.4",
pattern=f"mkdir -p {remote_mount}")
# No docker cp for file_mounts
process_runner.assert_not_has_call("1.2.3.4", pattern="docker cp")
process_runner.assert_has_call(
"1.2.3.4",
pattern=f"-avz {local_mount} [email protected]:{remote_host_mount}")
process_runner.clear_history()
##############################
process_runner.respond_to_call("docker inspect -f", ["true"])
cmd_runner.run_rsync_up(local_file,
remote_file,
options={"docker_mount_if_possible": False})
# Make sure we do not copy directly to raw destination
process_runner.assert_not_has_call(
"1.2.3.4", pattern=f"-avz {local_file} [email protected]:{remote_file}")
process_runner.assert_not_has_call("1.2.3.4",
pattern=f"mkdir -p {remote_file}")
process_runner.assert_has_call("1.2.3.4", pattern="docker cp")
process_runner.assert_has_call(
"1.2.3.4", pattern=f"-avz {local_file} [email protected]:{remote_host_file}")
process_runner.clear_history()
##############################
cmd_runner.run_rsync_down(remote_mount,
local_mount,
options={"docker_mount_if_possible": True})
process_runner.assert_not_has_call("1.2.3.4", pattern="docker cp")
process_runner.assert_not_has_call(
"1.2.3.4", pattern=f"-avz [email protected]:{remote_mount} {local_mount}")
process_runner.assert_has_call(
"1.2.3.4",
pattern=f"-avz [email protected]:{remote_host_mount} {local_mount}")
process_runner.clear_history()
##############################
cmd_runner.run_rsync_down(remote_file,
local_file,
options={"docker_mount_if_possible": False})
process_runner.assert_has_call("1.2.3.4", pattern="docker cp")
process_runner.assert_not_has_call(
"1.2.3.4", pattern=f"-avz [email protected]:{remote_file} {local_file}")
process_runner.assert_has_call(
"1.2.3.4", pattern=f"-avz [email protected]:{remote_host_file} {local_file}")
