def static_driver_fn(): global_rendezv = RendezvousServer(verbose=1) global_rendezv_port = global_rendezv.start() print("Rendezvous server started, port: " + str(global_rendezv_port)) # worker_list = "localhost:1" hosts = parse_hosts(worker_list) host_alloc_plan = get_host_assignments(hosts, 1) global_rendezv.init(host_alloc_plan) return (global_rendezv_port, host_alloc_plan)
class HorovodRendezvousServer(object): def __init__(self, server_host): self._rendezvous_host = server_host self._rendezvous_id = 0 self._worker_hosts = [] self._rendezvous_server = RendezvousServer(verbose=True) self._rendezvous_port = None def start(self): self._rendezvous_port = self._rendezvous_server.start() def set_worker_hosts(self, worker_hosts): """ Set worker hosts into RendezvousServer. Args: worker_hosts: List of host string. """ if sorted(worker_hosts) == sorted(self._worker_hosts): return self._rendezvous_id += 1 self._worker_hosts = worker_hosts host_alloc_plan = self._get_host_plan() self._rendezvous_server.init(host_alloc_plan) def _get_host_plan(self): hosts = [] for host in self._worker_hosts: hosts.append(host + ":" + str(_WORKER_SLOT_NUMBER)) host_infos = parse_hosts(_HOST_SEP.join(hosts)) host_alloc_plan = get_host_assignments(host_infos, len(host_infos)) return host_alloc_plan def get_rendezvous_host(self): return self._rendezvous_host def get_rendezvous_port(self): return self._rendezvous_port def get_worker_host_rank(self, host): # -1 if host not in worker_hosts list. if host not in self._worker_hosts: return -1 return self._worker_hosts.index(host) def get_size(self): return len(self._worker_hosts) def get_rendezvous_id(self): return self._rendezvous_id
def launch_gloo(command, exec_command, settings, nics, env, server_ip): """ Launches the given command multiple times using gloo. Each command is launched via exec_command. :param command: command to launch :param exec_command: means to execute a single command :param settings: settings for the distribution :param nics: common interfaces :param env: environment to use :param server_ip: ip to use for rendezvous server """ # Make the output directory if it does not exist if settings.output_filename: _mkdir_p(settings.output_filename) # start global rendezvous server and get port that it is listening on rendezvous = RendezvousServer(settings.verbose) # allocate processes into slots hosts = parse_hosts(settings.hosts) host_alloc_plan = get_host_assignments(hosts, settings.num_proc) # start global rendezvous server and get port that it is listening on pedl_provisioned_port = int( os.environ.get('PEDL_HOROVOD_GLOO_RENDEZVOUS_PORT', 0)) global_rendezv_port = rendezvous.start( pedl_provisioned_port=pedl_provisioned_port) rendezvous.init(host_alloc_plan) run_command = get_run_command(command, server_ip, nics, global_rendezv_port) slot_info_to_command = _slot_info_to_command_fn(run_command, env) event = register_shutdown_event() args_list = [[slot_info_to_command(slot_info), slot_info, [event]] for slot_info in host_alloc_plan] # If an error occurs in one thread, entire process will be terminated. # Otherwise, threads will keep running. res = threads.execute_function_multithreaded(exec_command, args_list, block_until_all_done=True) for name, value in sorted(res.items(), key=lambda item: item[1][1]): exit_code, timestamp = value if exit_code != 0: raise RuntimeError( 'Horovod detected that one or more processes exited with non-zero ' 'status, thus causing the job to be terminated. The first process ' 'to do so was:\nProcess name: {name}\nExit code: {code}\n'. format(name=name, code=exit_code))
def elastic_driver_fn(): global_rendezv = RendezvousServer(verbose=1) discover_hosts = discovery.HostDiscoveryScript( "/Users/zuston/iqiyiDev/horovod-opal/dis.sh", 3) driver = ElasticDriver(global_rendezv, discover_hosts, min_np=2, max_np=4) handler = create_rendezvous_handler(driver) global_rendezv_port = global_rendezv.start(handler) print('port: ' + str(global_rendezv_port)) print('wait for available slots: {}'.format(2)) current_hosts = driver.wait_for_available_slots(2) print("current hosts:" + str(current_hosts)) pending_slots = driver._update_host_assignments(current_hosts) print("pending hosts:" + str(pending_slots)) driver._worker_registry.reset(driver.world_size())
def static_driver_fn(): if is_in_test_mode: print("In unit test mode. fake port: " + fake_server_port) return (fake_server_port, get_host_assignments(parse_hosts(worker_list), 1)) global_rendezv = RendezvousServer(verbose=1) global_rendezv_port = global_rendezv.start() print("Rendezvous server started, port: " + str(global_rendezv_port)) # worker_list = "localhost:1" hosts = parse_hosts(worker_list) host_alloc_plan = get_host_assignments(hosts, 1) global_rendezv.init(host_alloc_plan) return (global_rendezv_port, host_alloc_plan)
class Coordinator: """Responsible for instantiating the Rendezvous server. Args: settings: Horovod Settings object.""" rendezvous = None global_rendezv_port = None nics = None node_id_by_rank = None def __init__( self, settings, ): self.settings = settings self.node_id_by_rank = defaultdict(list) self._hostnames = set() @property def world_size(self) -> int: return sum(len(ranks) for ranks in self.node_id_by_rank.values()) @property def hostnames(self): return self._hostnames @property def node_id_string(self) -> str: return ",".join([ f"{node_id}:{len(ranks)}" for node_id, ranks in self.node_id_by_rank.items() ]) def register(self, hostname: str, node_id: str, world_rank: int): self._hostnames.add(hostname) self.node_id_by_rank[node_id].append(world_rank) def finalize_registration(self) -> dict: """Return a dictionary for all ranks.""" rank_to_info = {} cross_sizes = defaultdict(int) cross_ranks = {} for rank_list in self.node_id_by_rank.values(): for local_rank, world_rank in enumerate(rank_list): cross_ranks[world_rank] = cross_sizes[local_rank] cross_sizes[local_rank] += 1 for node_world_rank, (node_id, ranks) in enumerate( self.node_id_by_rank.items()): for local_rank, world_rank in enumerate(ranks): rank_to_info[world_rank] = dict( HOROVOD_CROSS_RANK=cross_ranks[world_rank], HOROVOD_CROSS_SIZE=cross_sizes[local_rank], HOROVOD_LOCAL_RANK=local_rank, HOROVOD_LOCAL_SIZE=len(ranks)) return rank_to_info def establish_rendezvous(self) -> Dict[str, str]: """Creates the rendezvous server and identifies the nics to be used. Returns: Environment variables for each worker. """ # start global rendezvous server and get port that it is listening on self.rendezvous = RendezvousServer(self.settings.verbose) # allocate processes into slots # hosts = parse_hosts(hosts_string="10.11.11.11:4,10.11.11.12:4") parsed_node_ids = hosts.parse_hosts(hosts_string=self.node_id_string) host_alloc_plan = hosts.get_host_assignments(parsed_node_ids, self.world_size) # start global rendezvous server and get port that it is listening on self.global_rendezv_port = self.rendezvous.start() self.rendezvous.init(host_alloc_plan) return { # needs to be real address "HOROVOD_GLOO_RENDEZVOUS_ADDR": ray.util.get_node_ip_address(), "HOROVOD_GLOO_RENDEZVOUS_PORT": str(self.global_rendezv_port), "HOROVOD_CONTROLLER": "gloo", "HOROVOD_CPU_OPERATIONS": "gloo", }
class ElasticRayExecutor: """Executor for elastic jobs using Ray. Leverages the Ray global state to detect available hosts and slots. Assumes that the entire Ray cluster is available for the Executor to use. Args: settings: Configuration for the elastic job setup. You can use a standard Horovod ElasticSettings object or create one directly from ElasticRayExecutor.create_settings. use_gpu (bool): Whether to use GPU for allocation. cpus_per_slot (int): Number of CPU resources to allocate to each worker. gpus_per_slot (int): Number of GPU resources to allocate to each worker. env_vars (Dict): Environment variables to be set on the actors (worker processes) before initialization. override_discovery (bool): Whether for the ElasticRayExecutor to automatically provide a discovery mechanism for ElasticSettings. Example: .. code-block:: python import ray ray.init(address="auto") settings = ElasticRayExecutor.create_settings(verbose=True) executor = ElasticRayExecutor( settings, use_gpu=True, cpus_per_slot=2) executor.start() executor.run(train_fn) warning:: .. deprecated:: 0.25.0 """ @staticmethod def create_settings( min_num_proc: int = 1, max_num_proc: int = None, reset_limit: int = None, elastic_timeout: int = 600, timeout_s: int = 30, ssh_identity_file: str = None, nics: str = None, # min_np is deprecated, use min_num_proc instead min_np=None, # max_np is deprecated, use max_num_proc instead max_np=None, **kwargs): """Returns a Settings object for ElasticRayExecutor. Note that the `discovery` property will be set at runtime. Args: min_num_proc (int): Minimum number of processes running for training to continue. If number of available processes dips below this threshold, then training will wait for more instances to become available. max_num_proc (int): Maximum number of training processes, beyond which no additional processes will be created. If not specified, then will be unbounded. reset_limit (int): Maximum number of times that the training job can scale up or down the number of workers after which the job is terminated. elastic_timeout (int): Timeout for elastic initialisation after re-scaling the cluster. The default value is 600 seconds. Alternatively, the environment variable HOROVOD_ELASTIC_TIMEOUT can also be used.' timeout_s (int): Horovod performs all the checks and starts the processes before the specified timeout. The default value is 30 seconds. ssh_identity_file (str): File on the driver from which the identity (private key) is read. nics (set): Network interfaces that can be used for communication. """ if min_np is not None: min_num_proc = min_np warnings.warn('min_np is deprecated, use min_num_proc instead', DeprecationWarning) if max_np is not None: max_num_proc = max_np warnings.warn('max_np is deprecated, use max_num_proc instead', DeprecationWarning) start_timeout = timeout.Timeout( timeout_s, message="Timed out waiting for {activity}. Please " "check connectivity between servers. You " "may need to increase the --start-timeout " "parameter if you have too many servers.") ssh_identity_file = ssh_identity_file or os.path.expanduser( "~/ray_bootstrap_key.pem") settings = ElasticSettings( discovery=None, min_num_proc=min_num_proc, max_num_proc=max_num_proc, elastic_timeout=elastic_timeout, reset_limit=reset_limit, num_proc=min_num_proc, ssh_identity_file=ssh_identity_file, nics=nics, start_timeout=start_timeout, key=secret.make_secret_key() if secret else None, **kwargs) return settings def __init__(self, settings: ElasticSettings, use_gpu: bool = False, cpus_per_slot: int = 1, gpus_per_slot: Optional[int] = None, env_vars: dict = None, override_discovery=True): if gpus_per_slot and not use_gpu: raise ValueError("gpus_per_slot is set, but use_gpu is False. " "use_gpu must be True if gpus_per_slot is set. ") gpus_per_slot = gpus_per_slot or int(use_gpu) if use_gpu and gpus_per_slot < 1: raise ValueError( f"gpus_per_slot must be >= 1: Got {gpus_per_slot}.") if override_discovery: settings.discovery = RayHostDiscovery(use_gpu=use_gpu, cpus_per_slot=cpus_per_slot, gpus_per_slot=gpus_per_slot) self.cpus_per_slot = cpus_per_slot self.gpus_per_slot = gpus_per_slot self.use_gpu = use_gpu self.settings = settings self.driver = None self.rendezvous = None self.env_vars = env_vars or {} def start(self): """Starts the Horovod driver and services.""" self.rendezvous = RendezvousServer(self.settings.verbose) self.driver = ElasticDriver(rendezvous=self.rendezvous, discovery=self.settings.discovery, min_num_proc=self.settings.min_num_proc, max_num_proc=self.settings.max_num_proc, timeout=self.settings.elastic_timeout, reset_limit=self.settings.reset_limit, verbose=self.settings.verbose) handler = create_rendezvous_handler(self.driver) logger.debug("[ray] starting rendezvous") global_rendezv_port = self.rendezvous.start(handler) logger.debug(f"[ray] waiting for {self.settings.num_proc} to start.") self.driver.wait_for_available_slots(self.settings.num_proc) # Host-to-host common interface detection # requires at least 2 hosts in an elastic job. min_hosts = _get_min_start_hosts(self.settings) current_hosts = self.driver.wait_for_available_slots( self.settings.num_proc, min_hosts=min_hosts) logger.debug("[ray] getting common interfaces") nics = detect_nics( self.settings, all_host_names=current_hosts.host_assignment_order, ) logger.debug("[ray] getting driver IP") server_ip = socket.gethostbyname(socket.gethostname()) self.run_env_vars = create_run_env_vars(server_ip, nics, global_rendezv_port, elastic=True) def _create_resources(self, hostname: str): resources = dict(num_cpus=self.cpus_per_slot, num_gpus=int(self.use_gpu) * self.gpus_per_slot, resources={f"node:{hostname}": 0.01}) return resources def _create_remote_worker(self, slot_info, worker_env_vars): hostname = slot_info.hostname loaded_worker_cls = self.remote_worker_cls.options( **self._create_resources(hostname)) worker = loaded_worker_cls.remote() worker.update_env_vars.remote(worker_env_vars) worker.update_env_vars.remote(create_slot_env_vars(slot_info)) if self.use_gpu: visible_devices = ",".join( [str(i) for i in range(slot_info.local_size)]) worker.update_env_vars.remote( {"CUDA_VISIBLE_DEVICES": visible_devices}) return worker def _create_spawn_worker_fn(self, return_results: List, worker_fn: Callable, queue: "ray.util.Queue") -> Callable: self.remote_worker_cls = ray.remote(BaseHorovodWorker) # event = register_shutdown_event() worker_env_vars = {} worker_env_vars.update(self.run_env_vars.copy()) worker_env_vars.update(self.env_vars.copy()) worker_env_vars.update({"PYTHONUNBUFFERED": "1"}) def worker_loop(slot_info, events): def ping_worker(worker): # There is an odd edge case where a node can be removed # before the remote worker is started, leading to a failure # in trying to create the horovod mesh. try: ping = worker.execute.remote(lambda _: 1) ray.get(ping, timeout=10) except Exception as e: logger.error(f"{slot_info.hostname}: Ping failed - {e}") return False return True worker = self._create_remote_worker(slot_info, worker_env_vars) if not ping_worker(worker): return 1, time.time() ray.get(worker.set_queue.remote(queue)) future = worker.execute.remote(lambda _: worker_fn()) result = None while result is None: try: # TODO: make this event driven at some point. retval = ray.get(future, timeout=0.1) return_results.append((slot_info.rank, retval)) # Success result = 0, time.time() except GetTimeoutError: # Timeout if any(e.is_set() for e in events): ray.kill(worker) result = 1, time.time() except Exception as e: logger.error(f"{slot_info.hostname}[{slot_info.rank}]:{e}") ray.kill(worker) result = 1, time.time() logger.debug(f"Worker ({slot_info}) routine is done!") return result return worker_loop def run(self, worker_fn: Callable, callbacks: Optional[List[Callable]] = None) -> List[Any]: """Executes the provided function on all workers. Args: worker_fn: Target elastic function that can be executed. callbacks: List of callables. Each callback must either be a callable function or a class that implements __call__. Every callback will be invoked on every value logged by the rank 0 worker. Returns: List of return values from every completed worker. """ return_values = [] from ray.util.queue import Queue import inspect args = inspect.getfullargspec(Queue).args if "actor_options" not in args: # Ray 1.1 and less _queue = Queue() else: _queue = Queue(actor_options={ "num_cpus": 0, "resources": { ray.state.current_node_id(): 0.001 } }) self.driver.start( self.settings.num_proc, self._create_spawn_worker_fn(return_values, worker_fn, _queue)) def _process_calls(queue, callbacks, event): if not callbacks: return while queue.actor: if not queue.empty(): result = queue.get_nowait() for c in callbacks: c(result) # avoid slamming the CI elif event.is_set(): break time.sleep(0.1) try: event = threading.Event() _callback_thread = threading.Thread(target=_process_calls, args=(_queue, callbacks, event), daemon=True) _callback_thread.start() res = self.driver.get_results() event.set() if _callback_thread: _callback_thread.join(timeout=60) finally: if hasattr(_queue, "shutdown"): _queue.shutdown() else: done_ref = _queue.actor.__ray_terminate__.remote() done, not_done = ray.wait([done_ref], timeout=5) if not_done: ray.kill(_queue.actor) self.driver.stop() if res.error_message is not None: raise RuntimeError(res.error_message) for name, value in sorted(res.worker_results.items(), key=lambda item: item[1][1]): exit_code, timestamp = value if exit_code != 0: raise RuntimeError( 'Horovod detected that one or more processes ' 'exited with non-zero ' 'status, thus causing the job to be terminated. ' 'The first process ' 'to do so was:\nProcess name: {name}\nExit code: {code}\n'. format(name=name, code=exit_code)) return_values = [ value for k, value in sorted(return_values, key=lambda kv: kv[0]) ] return return_values
class ElasticRayExecutor: """Executor for elastic jobs using Ray. Leverages the Ray global state to detect available hosts and slots. Assumes that the entire Ray cluster is available for the Executor to use. Args: settings: Configuration for the elastic job setup. You can use a standard Horovod ElasticSettings object or create one directly from ElasticRayExecutor.create_settings. use_gpu (bool): Whether to use GPU for allocation. cpus_per_slot (int): Number of CPU resources to allocate to each worker. gpus_per_slot (int): Number of GPU resources to allocate to each worker. env_vars (Dict): Environment variables to be set on the actors (worker processes) before initialization. override_discovery (bool): Whether for the ElasticRayExecutor to automatically provide a discovery mechanism for ElasticSettings. Example: .. code-block:: python import ray ray.init(address="auto") settings = ElasticRayExecutor.create_settings(verbose=True) executor = ElasticRayExecutor( settings, use_gpu=True, cpus_per_slot=2) executor.start() executor.run(train_fn) """ @staticmethod def create_settings(min_np: int = 1, max_np: int = None, reset_limit: int = None, elastic_timeout: int = 600, timeout_s: int = 30, ssh_identity_file: str = None, nics: str = None, **kwargs): """Returns a Settings object for ElasticRayExecutor. Note that the `discovery` property will be set at runtime. Args: min_np (int): Minimum number of processes running for training to continue. If number of available processes dips below this threshold, then training will wait for more instances to become available. max_np (int): Maximum number of training processes, beyond which no additional processes will be created. If not specified, then will be unbounded. reset_limit (int): Maximum number of times that the training job can scale up or down the number of workers after which the job is terminated. elastic_timeout (int): Timeout for elastic initialisation after re-scaling the cluster. The default value is 600 seconds. Alternatively, the environment variable HOROVOD_ELASTIC_TIMEOUT can also be used.' timeout_s (int): Horovod performs all the checks and starts the processes before the specified timeout. The default value is 30 seconds. ssh_identity_file (str): File on the driver from which the identity (private key) is read. nics (set): Network interfaces that can be used for communication. """ start_timeout = timeout.Timeout( timeout_s, message="Timed out waiting for {activity}. Please " "check connectivity between servers. You " "may need to increase the --start-timeout " "parameter if you have too many servers.") ssh_identity_file = ssh_identity_file or os.path.expanduser( "~/ray_bootstrap_key.pem") settings = ElasticSettings( discovery=None, min_np=min_np, max_np=max_np, elastic_timeout=elastic_timeout, reset_limit=reset_limit, num_proc=min_np, ssh_identity_file=ssh_identity_file, nics=nics, start_timeout=start_timeout, key=secret.make_secret_key() if secret else None, **kwargs) return settings def __init__(self, settings: ElasticSettings, use_gpu: bool = False, cpus_per_slot: int = 1, gpus_per_slot: Optional[int] = None, env_vars: dict = None, override_discovery=True): if gpus_per_slot and not use_gpu: raise ValueError("gpus_per_slot is set, but use_gpu is False. " "use_gpu must be True if gpus_per_slot is set. ") if use_gpu and gpus_per_slot < 1: raise ValueError( f"gpus_per_slot must be >= 1: Got {gpus_per_slot}.") gpus_per_slot = gpus_per_slot or 1 if override_discovery: settings.discovery = RayHostDiscovery(use_gpu=use_gpu, cpus_per_slot=cpus_per_slot, gpus_per_slot=gpus_per_slot) self.cpus_per_slot = cpus_per_slot self.gpus_per_slot = gpus_per_slot self.use_gpu = use_gpu self.settings = settings self.driver = None self.rendezvous = None self.env_vars = env_vars or {} def start(self): """Starts the Horovod driver and services.""" self.rendezvous = RendezvousServer(self.settings.verbose) self.driver = ElasticDriver(rendezvous=self.rendezvous, discovery=self.settings.discovery, min_np=self.settings.min_np, max_np=self.settings.max_np, timeout=self.settings.elastic_timeout, reset_limit=self.settings.reset_limit, verbose=self.settings.verbose) handler = create_rendezvous_handler(self.driver) global_rendezv_port = self.rendezvous.start(handler) self.driver.wait_for_available_slots(self.settings.num_proc) # Host-to-host common interface detection # requires at least 2 hosts in an elastic job. min_hosts = _get_min_start_hosts(self.settings) current_hosts = self.driver.wait_for_available_slots( self.settings.num_proc, min_hosts=min_hosts) nics = driver_service.get_common_interfaces( self.settings, current_hosts.host_assignment_order) server_ip = network.get_driver_ip(nics) self.run_env_vars = create_run_env_vars(server_ip, nics, global_rendezv_port, elastic=True) def _create_resources(self, hostname: str): resources = dict(num_cpus=self.cpus_per_slot, num_gpus=int(self.use_gpu) * self.gpus_per_slot, resources={f"node:{hostname}": 0.01}) return resources def _create_remote_worker(self, slot_info, worker_env_vars): hostname = slot_info.hostname loaded_worker_cls = self.remote_worker_cls.options( **self._create_resources(hostname)) worker = loaded_worker_cls.remote() worker.update_env_vars.remote(worker_env_vars) worker.update_env_vars.remote(create_slot_env_vars(slot_info)) if self.use_gpu: visible_devices = ",".join( [str(i) for i in range(slot_info.local_size)]) worker.update_env_vars.remote( {"CUDA_VISIBLE_DEVICES": visible_devices}) return worker def _create_spawn_worker_fn(self, return_results: List, worker_fn: Callable) -> Callable: self.remote_worker_cls = ray.remote(BaseHorovodWorker) # event = register_shutdown_event() worker_env_vars = {} worker_env_vars.update(self.run_env_vars.copy()) worker_env_vars.update(self.env_vars.copy()) worker_env_vars.update({"PYTHONUNBUFFERED": "1"}) def worker_loop(slot_info, events): worker = self._create_remote_worker(slot_info, worker_env_vars) future = worker.execute.remote(lambda _: worker_fn()) result = None while result is None: try: # TODO: make this event driven at some point. retval = ray.get(future, timeout=0.1) return_results.append((slot_info.rank, retval)) # Success result = 0, time.time() except GetTimeoutError: # Timeout if any(e.is_set() for e in events): ray.kill(worker) result = 1, time.time() except Exception as e: logger.exception(str(e)) # Fail result = 1, time.time() return result return worker_loop def run(self, worker_fn: Callable) -> List[Any]: """Executes the provided function on all workers. Args: worker_fn: Target elastic function that can be executed. Returns: List of return values from every completed worker. """ return_values = [] self.driver.start( self.settings.num_proc, self._create_spawn_worker_fn(return_values, worker_fn)) res = self.driver.get_results() self.driver.stop() if res.error_message is not None: raise RuntimeError(res.error_message) for name, value in sorted(res.worker_results.items(), key=lambda item: item[1][1]): exit_code, timestamp = value if exit_code != 0: raise RuntimeError( 'Horovod detected that one or more processes ' 'exited with non-zero ' 'status, thus causing the job to be terminated. ' 'The first process ' 'to do so was:\nProcess name: {name}\nExit code: {code}\n'. format(name=name, code=exit_code)) return_values = [ value for k, value in sorted(return_values, key=lambda kv: kv[0]) ] return return_values
class ElasticAdapter(Adapter): """Adapter for executing Ray calls for elastic Horovod jobs. Args: settings (horovod.Settings): Configuration for job setup. You can use a standard Horovod Settings object or create one directly from RayExecutor.create_settings. min_workers (int): Minimum number of processes running for training to continue. If number of available processes dips below this threshold, then training will wait for more instances to become available. max_workers (int): Maximum number of training processes, beyond which no additional processes will be created. If not specified, then will be unbounded. reset_limit (int): Maximum number of times that the training job can scale up or down the number of workers after which the job is terminated. cooldown_range (Tuple[int, int]): Range(in seconds) a failing host will remain in blacklist. Example: cooldown_range=(10, 100) This sets the minimum cooldown period to 10 seconds, and the maximum cooldown period to 100 seconds. elastic_timeout (int): Timeout for elastic initialisation after re-scaling the cluster. The default value is 600 seconds. Alternatively, the environment variable HOROVOD_ELASTIC_TIMEOUT can also be used.' cpus_per_worker (int): Number of CPU resources to allocate to each worker. use_gpu (bool): Whether to use GPU for allocation. TODO: this can be removed. gpus_per_worker (int): Number of GPU resources to allocate to each worker. override_discovery (bool): Whether for the ElasticRayExecutor to automatically provide a discovery mechanism for ElasticSettings. """ def __init__(self, settings, min_workers: int, max_workers: Optional[int] = None, use_gpu: bool = False, cpus_per_worker: int = 1, gpus_per_worker: Optional[int] = None, override_discovery: bool = True, reset_limit: int = None, cooldown_range: Optional[Tuple[int, int]] = None, elastic_timeout: int = 600): self.settings = settings if override_discovery: settings.discovery = RayHostDiscovery( use_gpu=use_gpu, cpus_per_worker=cpus_per_worker, gpus_per_worker=gpus_per_worker) self.cpus_per_worker = cpus_per_worker self.gpus_per_worker = gpus_per_worker self.use_gpu = use_gpu # moved from settings self.min_workers = min_workers self.max_workers = max_workers self.num_workers = min_workers self.reset_limit = reset_limit self.cooldown_range = cooldown_range self.elastic_timeout = elastic_timeout self.driver = None self.rendezvous = None def start(self, executable_cls: type = None, executable_args: Optional[List] = None, executable_kwargs: Optional[Dict] = None, extra_env_vars: Optional[Dict] = None): """Starts the Horovod driver and services. Args: executable_cls (type): The class that will be created within an actor (BaseHorovodWorker). This will allow Horovod to establish its connections and set env vars. executable_args (List): Arguments to be passed into the worker class upon initialization. executable_kwargs (Dict): Keyword arguments to be passed into the worker class upon initialization. extra_env_vars (Dict): Environment variables to be set on the actors (worker processes) before initialization. """ self.rendezvous = RendezvousServer(self.settings.verbose) self.driver = ElasticDriver(rendezvous=self.rendezvous, discovery=self.settings.discovery, min_np=self.min_workers, max_np=self.max_workers, timeout=self.elastic_timeout, reset_limit=self.reset_limit, cooldown_range=self.cooldown_range, verbose=self.settings.verbose) handler = create_rendezvous_handler(self.driver) logger.debug("[ray] starting rendezvous") global_rendezv_port = self.rendezvous.start(handler) logger.debug(f"[ray] waiting for {self.num_workers} to start.") self.driver.wait_for_available_slots(self.num_workers) # Host-to-host common interface detection # requires at least 2 hosts in an elastic job. min_hosts = _get_min_start_hosts(self.settings) current_hosts = self.driver.wait_for_available_slots( self.num_workers, min_hosts=min_hosts) logger.debug("[ray] getting common interfaces") nics = detect_nics( self.settings, all_host_names=current_hosts.host_assignment_order, ) logger.debug("[ray] getting driver IP") server_ip = socket.gethostbyname(socket.gethostname()) self.run_env_vars = create_run_env_vars(server_ip, nics, global_rendezv_port, elastic=True) self.executable_cls = executable_cls self.executable_args = executable_args self.executable_kwargs = executable_kwargs self.env_vars = extra_env_vars or {} def _create_resources(self, hostname: str): resources = dict(num_cpus=self.cpus_per_worker, num_gpus=int(self.use_gpu) * self.gpus_per_worker, resources={f"node:{hostname}": 0.01}) return resources def _create_remote_worker(self, slot_info, worker_env_vars): hostname = slot_info.hostname loaded_worker_cls = self.remote_worker_cls.options( **self._create_resources(hostname)) worker = loaded_worker_cls.remote() worker.update_env_vars.remote(worker_env_vars) worker.update_env_vars.remote(create_slot_env_vars(slot_info)) if self.use_gpu: visible_devices = ",".join( [str(i) for i in range(slot_info.local_size)]) worker.update_env_vars.remote( {"CUDA_VISIBLE_DEVICES": visible_devices}) return worker def _create_spawn_worker_fn(self, return_results: List, worker_fn: Callable, queue: "ray.util.Queue") -> Callable: self.remote_worker_cls = ray.remote(BaseHorovodWorker) # event = register_shutdown_event() worker_env_vars = {} worker_env_vars.update(self.run_env_vars.copy()) worker_env_vars.update(self.env_vars.copy()) worker_env_vars.update({"PYTHONUNBUFFERED": "1"}) def worker_loop(slot_info, events): def ping_worker(worker): # There is an odd edge case where a node can be removed # before the remote worker is started, leading to a failure # in trying to create the horovod mesh. try: ping = worker.execute.remote(lambda _: 1) ray.get(ping, timeout=10) except Exception as e: logger.error(f"{slot_info.hostname}: Ping failed - {e}") return False return True worker = self._create_remote_worker(slot_info, worker_env_vars) if not ping_worker(worker): return 1, time.time() ray.get(worker.set_queue.remote(queue)) future = worker.execute.remote(worker_fn) result = None while result is None: try: # TODO: make this event driven at some point. retval = ray.get(future, timeout=0.1) return_results.append((slot_info.rank, retval)) # Success result = 0, time.time() except GetTimeoutError: # Timeout if any(e.is_set() for e in events): ray.kill(worker) result = 1, time.time() except Exception as e: logger.error(f"{slot_info.hostname}[{slot_info.rank}]:{e}") ray.kill(worker) result = 1, time.time() logger.debug(f"Worker ({slot_info}) routine is done!") return result return worker_loop def run(self, fn: Callable[[Any], Any], args: Optional[List] = None, kwargs: Optional[Dict] = None, callbacks: Optional[List[Callable]] = None) -> List[Any]: """Executes the provided function on all workers. Args: fn: Target function that can be executed with arbitrary args and keyword arguments. args: List of arguments to be passed into the target function. kwargs: Dictionary of keyword arguments to be passed into the target function. callbacks: List of callables. Each callback must either be a callable function or a class that implements __call__. Every callback will be invoked on every value logged by the rank 0 worker. Returns: Deserialized return values from the target function. """ args = args or [] kwargs = kwargs or {} f = lambda _: fn(*args, **kwargs) return self._run_remote(f, callbacks=callbacks) def _run_remote(self, worker_fn: Callable, callbacks: Optional[List[Callable]] = None) -> List[Any]: """Executes the provided function on all workers. Args: worker_fn: Target elastic function that can be executed. callbacks: List of callables. Each callback must either be a callable function or a class that implements __call__. Every callback will be invoked on every value logged by the rank 0 worker. Returns: List of return values from every completed worker. """ return_values = [] from ray.util.queue import Queue import inspect args = inspect.getfullargspec(Queue).args if "actor_options" not in args: # Ray 1.1 and less _queue = Queue() else: _queue = Queue(actor_options={ "num_cpus": 0, "resources": { ray.state.current_node_id(): 0.001 } }) self.driver.start( self.num_workers, self._create_spawn_worker_fn(return_values, worker_fn, _queue)) def _process_calls(queue, callbacks, event): if not callbacks: return while queue.actor: if not queue.empty(): result = queue.get_nowait() for c in callbacks: c(result) # avoid slamming the CI elif event.is_set(): break time.sleep(0.1) try: event = threading.Event() _callback_thread = threading.Thread(target=_process_calls, args=(_queue, callbacks, event), daemon=True) _callback_thread.start() res = self.driver.get_results() event.set() if _callback_thread: _callback_thread.join(timeout=60) finally: if hasattr(_queue, "shutdown"): _queue.shutdown() else: done_ref = _queue.actor.__ray_terminate__.remote() done, not_done = ray.wait([done_ref], timeout=5) if not_done: ray.kill(_queue.actor) self.driver.stop() if res.error_message is not None: raise RuntimeError(res.error_message) for name, value in sorted(res.worker_results.items(), key=lambda item: item[1][1]): exit_code, timestamp = value if exit_code != 0: raise RuntimeError( 'Horovod detected that one or more processes ' 'exited with non-zero ' 'status, thus causing the job to be terminated. ' 'The first process ' 'to do so was:\nProcess name: {name}\nExit code: {code}\n'. format(name=name, code=exit_code)) return_values = [ value for k, value in sorted(return_values, key=lambda kv: kv[0]) ] return return_values def run_remote(self, fn: Callable[[Any], Any]) -> List[Any]: raise NotImplementedError( "ObjectRefs cannot be returned from Elastic runs as the workers are ephemeral" ) def execute(self, fn: Callable[["executable_cls"], Any], callbacks: Optional[List[Callable]] = None) -> List[Any]: """Executes the provided function on all workers. Args: fn: Target function to be invoked on every object. callbacks: List of callables. Each callback must either be a callable function or a class that implements __call__. Every callback will be invoked on every value logged by the rank 0 worker. Returns: Deserialized return values from the target function. """ return ray.get(self._run_remote(fn, callbacks=callbacks)) def execute_single(self, fn: Callable[["executable_cls"], Any]) -> List[Any]: """Executes the provided function on the rank 0 worker (chief). Args: fn: Target function to be invoked on the chief object. Returns: Deserialized return values from the target function. """ raise NotImplementedError( "Elastic mode does not support execute_single. Please use the execute method instead" ) def shutdown(self): """Destroys the driver.""" if not self.driver: return assert self.driver.finished() self.driver = None
class HorovodRendezvousServer(object): def __init__(self, server_host): self._rendezvous_host = server_host self._rendezvous_id = 0 self._worker_hosts = [] self._rendezvous_server = RendezvousServer(verbose=True) self._rendezvous_port = None self._next_worker_hosts = None self._ready_worker_hosts = set() self._rendezvous_completed = True self._lock = Lock() def start(self): self._rendezvous_port = self._rendezvous_server.start() def set_worker_hosts(self, worker_hosts): """ Set worker hosts into RendezvousServer. Args: worker_hosts: List of host string. """ if sorted(worker_hosts) != sorted(self._worker_hosts): self._next_worker_hosts = worker_hosts def _init_rendezvous_server(self): self._worker_hosts = self._next_worker_hosts self._next_worker_hosts = None host_alloc_plan = self._get_host_plan() self._rendezvous_server.init(host_alloc_plan) self._rendezvous_id += 1 self._rendezvous_completed = False def _get_host_plan(self): hosts = [] for host in self._worker_hosts: hosts.append(host + ":" + str(_WORKER_SLOT_NUMBER)) host_infos = parse_hosts(_HOST_SEP.join(hosts)) host_alloc_plan = get_host_assignments(host_infos, len(host_infos)) return host_alloc_plan def get_rendezvous_host(self): return self._rendezvous_host def get_rendezvous_port(self): return self._rendezvous_port def get_worker_host_rank(self, host): with self._lock: if self._next_worker_hosts and self._rendezvous_completed: time.sleep(2) # Wait 2s for workers to complete rendezvous. self._init_rendezvous_server() # -1 if host not in worker_hosts list. if host not in self._worker_hosts: return -1 if not self._rendezvous_completed: self._ready_worker_hosts.add(host) # If all active workers in the rendezvous are ready, # the server can start to set hosts for the next rendezvous if self._ready_worker_hosts == set(self._worker_hosts): self._rendezvous_completed = True self._ready_worker_hosts = set() return self._worker_hosts.index(host) def get_size(self): return len(self._worker_hosts) def get_rendezvous_id(self): return self._rendezvous_id
def test_worker_notification_manager(self): """Tests that host add events are sent to the worker notification service and consumed.""" slots = {'host-1': 2} discovery = FixedHosts(slots) rendezvous = RendezvousServer() driver = ElasticDriver(rendezvous, discovery, min_np=2, max_np=4) driver.wait_for_available_slots(min_np=2) handler = create_rendezvous_handler(driver) common_intfs = network.get_local_intfs() addr = network.get_driver_ip(common_intfs) port = rendezvous.start(handler) nic = list(common_intfs)[0] rank_results = {} class NotificationReceiver: def __init__(self): self.events = [] def on_hosts_updated(self, timestamp, res): self.events.append((timestamp, res)) def add_host(): slots = {'host-1': 2, 'host-2': 2} discovery.set(slots) def remove_host(): slots = {'host-2': 2} discovery.set(slots) def exec_command(slot_info, events): manager = WorkerNotificationManager() manager.init(rendezvous_addr=addr, rendezvous_port=port, nic=nic, hostname=slot_info.hostname, local_rank=slot_info.local_rank) notification_receiver = NotificationReceiver() manager.register_listener(notification_receiver) driver.record_ready(slot_info.hostname, slot_info.local_rank) if slot_info.rank == 0: add_host() driver.wait_for_available_slots(4) if slot_info.rank == 0: remove_host() # Busy wait for the number of available slots to decrease while driver._host_manager.current_hosts.count_available_slots( ) > 2: time.sleep(0.01) rank_results[slot_info.rank] = notification_receiver.events return 0, time.time() driver.start(np=2, create_worker_fn=exec_command) res = driver.get_results().worker_results driver.stop() assert len(res) == 2 for name, (exit_code, timestamp) in res.items(): assert exit_code == 0, name assert len(rank_results) == 2 for rank, events in rank_results.items(): expected = 2 if rank == 0 else 0 assert len(events) == expected, rank if rank == 0: # First update is an add assert events[0][1] == HostUpdateResult.added # Second update is a removal assert events[1][1] == HostUpdateResult.removed rendezvous.stop()
class HorovodRendezvousServer(object): """The rendezvous server can collect worker hosts (ip) to help these workers to build an AllReduce ring using `hvd.init`. The state transition diagram of the server is: |------------------| | start | |next_hosts = None | |------------------| | worker-0 sends the start | message | |------------------| |-- |next_hosts = [0] |------------------| | |------------------| | worker-1 sends | worker-0 queries | the start message| a rank | |---------------------| worker-0 queries |--------------------| |next_hosts = [0, 1] | a rank |cur_hosts=next_hosts| | | ----------------> |next_hosts=None | |---------------------| | ready_hosts adds | | the worker |<---| |<--------- |--------------------| | worker-2 sends | | the start message | worker-2 quries | |-------------------------| a rank and | |next_hosts=cur_hosts+[2] | ready_hosts=cur_hosts | | ------------------------| ----------------------->| """ def __init__(self, server_host): self._rendezvous_host = server_host self._init_attributes() def _init_attributes(self): self._rendezvous_id = 0 self._cur_rendezvous_hosts = [] self._rendezvous_server = RendezvousServer(verbose=True) self._rendezvous_port = None self._next_rendezvous_hosts = None self._ready_worker_hosts = set() self._cur_rendezvous_completed = True self._lock = Lock() def start(self): self._rendezvous_port = self._rendezvous_server.start() def _init_rendezvous_server(self): logger.info("Initialize rendezvous server with hosts {}".format( self._next_rendezvous_hosts)) self._cur_rendezvous_hosts = self._next_rendezvous_hosts self._next_rendezvous_hosts = None host_alloc_plan = self._get_host_plan() self._rendezvous_server.init(host_alloc_plan) self._rendezvous_id += 1 self._cur_rendezvous_completed = False def _get_host_plan(self): hosts = [] for host in self._cur_rendezvous_hosts: hosts.append(host + ":" + str(_WORKER_SLOT_NUMBER)) host_infos = parse_hosts(_HOST_SEP.join(hosts)) host_alloc_plan = get_host_assignments(host_infos, len(host_infos)) return host_alloc_plan def get_rendezvous_host(self): return self._rendezvous_host def get_rendezvous_port(self): return self._rendezvous_port def get_worker_host_rank(self, host): with self._lock: if self._next_rendezvous_hosts and self._cur_rendezvous_completed: time.sleep(2) # Wait 2s for workers to complete rendezvous. self._init_rendezvous_server() # -1 if host not in worker_hosts list. if host not in self._cur_rendezvous_hosts: return -1 if not self._cur_rendezvous_completed: self._ready_worker_hosts.add(host) # If all active workers in the rendezvous are ready, # the server can start to set hosts for the next rendezvous if self._ready_worker_hosts == set(self._cur_rendezvous_hosts): self._cur_rendezvous_completed = True self._ready_worker_hosts = set() return self._cur_rendezvous_hosts.index(host) def get_size(self): return len(self._cur_rendezvous_hosts) def get_rendezvous_id(self): return self._rendezvous_id def add_worker(self, worker_host): with self._lock: logger.info( "Add worker host {} into rendenzvous and cur hosts {}.".format( worker_host, self._cur_rendezvous_hosts)) if worker_host: if self._next_rendezvous_hosts is None: self._next_rendezvous_hosts = copy.deepcopy( self._cur_rendezvous_hosts) # Master will not add any worker if the current rendezvous # hosts become empty after starting training. if self._rendezvous_id > 0 and not self._next_rendezvous_hosts: return if worker_host not in self._next_rendezvous_hosts: self._next_rendezvous_hosts.append(worker_host) def remove_worker(self, worker_host): with self._lock: logger.info( "Remove worker host {} from rendenzvous.".format(worker_host)) if worker_host in self._cur_rendezvous_hosts: if self._next_rendezvous_hosts is None: self._next_rendezvous_hosts = copy.deepcopy( self._cur_rendezvous_hosts) self._next_rendezvous_hosts.pop( self._next_rendezvous_hosts.index(worker_host))
class Coordinator: """Responsible for instantiating the Rendezvous server. Args: settings: Horovod Settings object.""" rendezvous = None global_rendezv_port = None nics = None hostnames = None def __init__( self, settings, ): self.settings = settings self.hostnames_by_rank = defaultdict(list) @property def world_size(self) -> int: return sum(len(ranks) for ranks in self.hostnames_by_rank.values()) @property def hoststring(self) -> str: return ",".join([ f"{host}:{len(ranks)}" for host, ranks in self.hostnames_by_rank.items() ]) def register(self, hostname: str, world_rank: int): self.hostnames_by_rank[hostname].append(world_rank) def finalize_registration(self) -> dict: """Return a dictionary for all ranks.""" rank_to_info = {} for node_world_rank, (hostname, ranks) in enumerate( self.hostnames_by_rank.items()): for local_rank, world_rank in enumerate(ranks): rank_to_info[world_rank] = dict( NODE_WORLD_RANK=node_world_rank, NODE_WORLD_SIZE=len(self.hostnames_by_rank), LOCAL_RANK=local_rank, LOCAL_SIZE=len(ranks)) return rank_to_info def establish_rendezvous(self) -> Dict[str, str]: """Creates the rendezvous server and identifies the nics to be used. Returns: Environment variables for each worker. """ # start global rendezvous server and get port that it is listening on self.rendezvous = RendezvousServer(self.settings.verbose) # allocate processes into slots # hosts = parse_hosts(hosts_string="10.11.11.11:4,10.11.11.12:4") parsed_hosts = hosts.parse_hosts(hosts_string=self.hoststring) host_alloc_plan = hosts.get_host_assignments(parsed_hosts, self.world_size) # start global rendezvous server and get port that it is listening on self.global_rendezv_port = self.rendezvous.start() self.rendezvous.init(host_alloc_plan) # remote_host_names = network.filter_local_addresses() self.nics = driver_service.get_common_interfaces( self.settings, list(self.hostnames_by_rank)) return { "HOROVOD_GLOO_RENDEZVOUS_ADDR": ray.services.get_node_ip_address(), "HOROVOD_GLOO_RENDEZVOUS_PORT": str(self.global_rendezv_port), "HOROVOD_CONTROLLER": "gloo", "HOROVOD_CPU_OPERATIONS": "gloo", "HOROVOD_GLOO_IFACE": str(list(self.nics)[0]), # TODO "NCCL_SOCKET_IFNAME": ",".join(self.nics), # TDOO }