def detect_sync_to_driver(
sync_to_driver: Union[None, bool, Type],
cluster_config_file: str = "~/ray_bootstrap_config.yaml"):
from ray.tune.integration.docker import DockerSyncer
if isinstance(sync_to_driver, Type):
return sync_to_driver
elif isinstance(sync_to_driver, bool) and sync_to_driver is False:
return sync_to_driver
# Else: True or None. Auto-detect.
cluster_config_file = os.path.expanduser(cluster_config_file)
if not os.path.exists(cluster_config_file):
return sync_to_driver
with open(cluster_config_file, "rt") as fp:
config = yaml.safe_load(fp.read())
if config.get("docker"):
logger.debug(
"Detected docker autoscaling environment. Using `DockerSyncer` "
"as sync client. If this is not correct or leads to errors, "
"please pass a `sync_to_driver` parameter in the `SyncConfig` to "
"`tune.run().` to manually configure syncing behavior.")
return DockerSyncer
if config.get("provider", {}).get("type", "") == "kubernetes":
from ray.tune.integration.kubernetes import (
NamespacedKubernetesSyncer, try_import_kubernetes)
if not try_import_kubernetes():
logger.warning(
"Detected Ray autoscaling environment on Kubernetes, "
"but Kubernetes Python CLI is not installed. "
"Checkpoint syncing may not work properly across "
"multiple pods. Be sure to install 'kubernetes' on "
"each container.")
namespace = config["provider"].get("namespace", "ray")
logger.debug(
f"Detected Ray autoscaling environment on Kubernetes. Using "
f"`NamespacedKubernetesSyncer` with namespace `{namespace}` "
f"as sync client. If this is not correct or leads to errors, "
f"please pass a `sync_to_driver` parameter in the `SyncConfig` "
f"to `tune.run()` to manually configure syncing behavior..")
return NamespacedKubernetesSyncer(namespace)
return sync_to_driver
def detect_cluster_syncer(
sync_config: Optional[SyncConfig],
cluster_config_file: str = "~/ray_bootstrap_config.yaml",
) -> Union[bool, Type, NodeSyncer]:
"""Detect cluster Syncer given SyncConfig.
Returns False if cloud checkpointing is enabled (when upload dir is
set).
Else, returns sync config syncer if manually specified.
Else, detects cluster environment (e.g. Docker, Kubernetes) and returns
syncer accordingly.
"""
from ray.tune.integration.docker import DockerSyncer
sync_config = sync_config or SyncConfig()
if bool(sync_config.upload_dir) or sync_config.syncer is None:
# No sync to driver for cloud checkpointing or if manually disabled
return False
_syncer = sync_config.syncer
if _syncer == "auto":
_syncer = None
if isinstance(_syncer, Type):
return _syncer
# Else: True or None. Auto-detect.
cluster_config_file = os.path.expanduser(cluster_config_file)
if not os.path.exists(cluster_config_file):
return _syncer
with open(cluster_config_file, "rt") as fp:
config = yaml.safe_load(fp.read())
if config.get("docker"):
logger.debug(
"Detected docker autoscaling environment. Using `DockerSyncer` "
"as sync client. If this is not correct or leads to errors, "
"please pass a `syncer` parameter in the `SyncConfig` to "
"`tune.run().` to manually configure syncing behavior.")
return DockerSyncer
if config.get("provider", {}).get("type", "") == "kubernetes":
from ray.tune.integration.kubernetes import (
NamespacedKubernetesSyncer,
try_import_kubernetes,
)
if not try_import_kubernetes():
logger.warning(
"Detected Ray autoscaling environment on Kubernetes, "
"but Kubernetes Python CLI is not installed. "
"Checkpoint syncing may not work properly across "
"multiple pods. Be sure to install 'kubernetes' on "
"each container.")
namespace = config["provider"].get("namespace", "ray")
logger.debug(
f"Detected Ray autoscaling environment on Kubernetes. Using "
f"`NamespacedKubernetesSyncer` with namespace `{namespace}` "
f"as sync client. If this is not correct or leads to errors, "
f"please pass a `syncer` parameter in the `SyncConfig` "
f"to `tune.run()` to manually configure syncing behavior..")
return NamespacedKubernetesSyncer(namespace)
return _syncer