def _check_if_container_restart_is_needed(
self, image: str, cleaned_bind_mounts: Dict[str, str]
) -> bool:
re_init_required = False
running_image = (
self.run(
check_docker_image(self.container_name, self.docker_cmd),
with_output=True,
run_env="host",
)
.decode("utf-8")
.strip()
)
if running_image != image:
cli_logger.error(
"A container with name {} is running image {} instead "
+ "of {} (which was provided in the YAML)",
self.container_name,
running_image,
image,
)
mounts = (
self.run(
check_bind_mounts_cmd(self.container_name, self.docker_cmd),
with_output=True,
run_env="host",
)
.decode("utf-8")
.strip()
)
try:
active_mounts = json.loads(mounts)
active_remote_mounts = {
mnt["Destination"].strip("/") for mnt in active_mounts
}
# Ignore ray bootstrap files.
requested_remote_mounts = {
self._docker_expand_user(remote).strip("/")
for remote in cleaned_bind_mounts.keys()
}
unfulfilled_mounts = requested_remote_mounts - active_remote_mounts
if unfulfilled_mounts:
re_init_required = True
cli_logger.warning(
"This Docker Container is already running. "
"Restarting the Docker container on "
"this node to pick up the following file_mounts {}",
unfulfilled_mounts,
)
except json.JSONDecodeError:
cli_logger.verbose(
"Unable to check if file_mounts specified in the YAML "
"differ from those on the running container."
)
return re_init_required
def run_init(self, *, as_head, file_mounts):
BOOTSTRAP_MOUNTS = [
"~/ray_bootstrap_config.yaml", "~/ray_bootstrap_key.pem"
]
image = self.docker_config.get("image")
image = self.docker_config.get(
f"{'head' if as_head else 'worker'}_image", image)
self._check_docker_installed()
if self.docker_config.get("pull_before_run", True):
assert image, "Image must be included in config if " + \
"pull_before_run is specified"
self.run("docker pull {}".format(image), run_env="host")
# Bootstrap files cannot be bind mounted because docker opens the
# underlying inode. When the file is switched, docker becomes outdated.
cleaned_bind_mounts = file_mounts.copy()
for mnt in BOOTSTRAP_MOUNTS:
cleaned_bind_mounts.pop(mnt, None)
if not self._check_container_status():
# Get home directory
image_env = self.ssh_command_runner.run(
"docker inspect -f '{{json .Config.Env}}' " + image,
with_output=True).decode().strip()
home_directory = "/root"
for env_var in json.loads(image_env):
if env_var.startswith("HOME="):
home_directory = env_var.split("HOME=")[1]
break
start_command = docker_start_cmds(
self.ssh_command_runner.ssh_user, image, cleaned_bind_mounts,
self.container_name,
self.docker_config.get("run_options", []) +
self.docker_config.get(
f"{'head' if as_head else 'worker'}_run_options", []) +
self._configure_runtime() + self._auto_configure_shm(),
self.ssh_command_runner.cluster_name, home_directory)
self.run(start_command, run_env="host")
else:
running_image = self.run(check_docker_image(self.container_name),
with_output=True,
run_env="host").decode("utf-8").strip()
if running_image != image:
logger.error(f"A container with name {self.container_name} " +
f"is running image {running_image} instead " +
f"of {image} (which was provided in the YAML")
mounts = self.run(check_bind_mounts_cmd(self.container_name),
with_output=True,
run_env="host").decode("utf-8").strip()
try:
active_mounts = json.loads(mounts)
active_remote_mounts = [
mnt["Destination"] for mnt in active_mounts
]
# Ignore ray bootstrap files.
for remote, local in cleaned_bind_mounts.items():
remote = self._docker_expand_user(remote)
if remote not in active_remote_mounts:
cli_logger.error(
"Please ray stop & restart cluster to "
f"allow mount {remote}:{local} to take hold")
except json.JSONDecodeError:
cli_logger.verbose(
"Unable to check if file_mounts specified in the YAML "
"differ from those on the running container.")
# Explicitly copy in ray bootstrap files.
for mount in BOOTSTRAP_MOUNTS:
if mount in file_mounts:
self.ssh_command_runner.run(
"docker cp {src} {container}:{dst}".format(
src=os.path.join(
self._get_docker_host_mount_location(
self.ssh_command_runner.cluster_name), mount),
container=self.container_name,
dst=self._docker_expand_user(mount)))
self.initialized = True
