def detect_sync_to_driver( sync_to_driver: Union[None, bool, Type], cluster_config_file: str = "~/ray_bootstrap_config.yaml"): from ray.tune.integration.docker import DockerSyncer if isinstance(sync_to_driver, Type): return sync_to_driver elif isinstance(sync_to_driver, bool) and sync_to_driver is False: return sync_to_driver # Else: True or None. Auto-detect. cluster_config_file = os.path.expanduser(cluster_config_file) if not os.path.exists(cluster_config_file): return sync_to_driver with open(cluster_config_file, "rt") as fp: config = yaml.safe_load(fp.read()) if config.get("docker"): logger.debug( "Detected docker autoscaling environment. Using `DockerSyncer` " "as sync client. If this is not correct or leads to errors, " "please pass a `sync_to_driver` parameter in the `SyncConfig` to " "`tune.run().` to manually configure syncing behavior.") return DockerSyncer if config.get("provider", {}).get("type", "") == "kubernetes": from ray.tune.integration.kubernetes import ( NamespacedKubernetesSyncer, try_import_kubernetes) if not try_import_kubernetes(): logger.warning( "Detected Ray autoscaling environment on Kubernetes, " "but Kubernetes Python CLI is not installed. " "Checkpoint syncing may not work properly across " "multiple pods. Be sure to install 'kubernetes' on " "each container.") namespace = config["provider"].get("namespace", "ray") logger.debug( f"Detected Ray autoscaling environment on Kubernetes. Using " f"`NamespacedKubernetesSyncer` with namespace `{namespace}` " f"as sync client. If this is not correct or leads to errors, " f"please pass a `sync_to_driver` parameter in the `SyncConfig` " f"to `tune.run()` to manually configure syncing behavior..") return NamespacedKubernetesSyncer(namespace) return sync_to_driver
def detect_cluster_syncer( sync_config: Optional[SyncConfig], cluster_config_file: str = "~/ray_bootstrap_config.yaml", ) -> Union[bool, Type, NodeSyncer]: """Detect cluster Syncer given SyncConfig. Returns False if cloud checkpointing is enabled (when upload dir is set). Else, returns sync config syncer if manually specified. Else, detects cluster environment (e.g. Docker, Kubernetes) and returns syncer accordingly. """ from ray.tune.integration.docker import DockerSyncer sync_config = sync_config or SyncConfig() if bool(sync_config.upload_dir) or sync_config.syncer is None: # No sync to driver for cloud checkpointing or if manually disabled return False _syncer = sync_config.syncer if _syncer == "auto": _syncer = None if isinstance(_syncer, Type): return _syncer # Else: True or None. Auto-detect. cluster_config_file = os.path.expanduser(cluster_config_file) if not os.path.exists(cluster_config_file): return _syncer with open(cluster_config_file, "rt") as fp: config = yaml.safe_load(fp.read()) if config.get("docker"): logger.debug( "Detected docker autoscaling environment. Using `DockerSyncer` " "as sync client. If this is not correct or leads to errors, " "please pass a `syncer` parameter in the `SyncConfig` to " "`tune.run().` to manually configure syncing behavior.") return DockerSyncer if config.get("provider", {}).get("type", "") == "kubernetes": from ray.tune.integration.kubernetes import ( NamespacedKubernetesSyncer, try_import_kubernetes, ) if not try_import_kubernetes(): logger.warning( "Detected Ray autoscaling environment on Kubernetes, " "but Kubernetes Python CLI is not installed. " "Checkpoint syncing may not work properly across " "multiple pods. Be sure to install 'kubernetes' on " "each container.") namespace = config["provider"].get("namespace", "ray") logger.debug( f"Detected Ray autoscaling environment on Kubernetes. Using " f"`NamespacedKubernetesSyncer` with namespace `{namespace}` " f"as sync client. If this is not correct or leads to errors, " f"please pass a `syncer` parameter in the `SyncConfig` " f"to `tune.run()` to manually configure syncing behavior..") return NamespacedKubernetesSyncer(namespace) return _syncer