def start(self): """Starts the Horovod driver and services.""" self.rendezvous = RendezvousServer(self.settings.verbose) self.driver = ElasticDriver(rendezvous=self.rendezvous, discovery=self.settings.discovery, min_np=self.settings.min_np, max_np=self.settings.max_np, timeout=self.settings.elastic_timeout, reset_limit=self.settings.reset_limit, verbose=self.settings.verbose) handler = create_rendezvous_handler(self.driver) global_rendezv_port = self.rendezvous.start(handler) self.driver.wait_for_available_slots(self.settings.num_proc) # Host-to-host common interface detection # requires at least 2 hosts in an elastic job. min_hosts = _get_min_start_hosts(self.settings) current_hosts = self.driver.wait_for_available_slots( self.settings.num_proc, min_hosts=min_hosts) nics = driver_service.get_common_interfaces( self.settings, current_hosts.host_assignment_order) server_ip = network.get_driver_ip(nics) self.run_env_vars = create_run_env_vars(server_ip, nics, global_rendezv_port, elastic=True)
def start(self): """Starts the Horovod driver and services.""" self.rendezvous = RendezvousServer(self.settings.verbose) self.driver = ElasticDriver(rendezvous=self.rendezvous, discovery=self.settings.discovery, min_num_proc=self.settings.min_num_proc, max_num_proc=self.settings.max_num_proc, timeout=self.settings.elastic_timeout, reset_limit=self.settings.reset_limit, verbose=self.settings.verbose) handler = create_rendezvous_handler(self.driver) logger.debug("[ray] starting rendezvous") global_rendezv_port = self.rendezvous.start(handler) logger.debug(f"[ray] waiting for {self.settings.num_proc} to start.") self.driver.wait_for_available_slots(self.settings.num_proc) # Host-to-host common interface detection # requires at least 2 hosts in an elastic job. min_hosts = _get_min_start_hosts(self.settings) current_hosts = self.driver.wait_for_available_slots( self.settings.num_proc, min_hosts=min_hosts) logger.debug("[ray] getting common interfaces") nics = detect_nics( self.settings, all_host_names=current_hosts.host_assignment_order, ) logger.debug("[ray] getting driver IP") server_ip = socket.gethostbyname(socket.gethostname()) self.run_env_vars = create_run_env_vars(server_ip, nics, global_rendezv_port, elastic=True)
def start(self, executable_cls: type = None, executable_args: Optional[List] = None, executable_kwargs: Optional[Dict] = None, extra_env_vars: Optional[Dict] = None): """Starts the Horovod driver and services. Args: executable_cls (type): The class that will be created within an actor (BaseHorovodWorker). This will allow Horovod to establish its connections and set env vars. executable_args (List): Arguments to be passed into the worker class upon initialization. executable_kwargs (Dict): Keyword arguments to be passed into the worker class upon initialization. extra_env_vars (Dict): Environment variables to be set on the actors (worker processes) before initialization. """ self.rendezvous = RendezvousServer(self.settings.verbose) self.driver = ElasticDriver(rendezvous=self.rendezvous, discovery=self.settings.discovery, min_np=self.min_workers, max_np=self.max_workers, timeout=self.elastic_timeout, reset_limit=self.reset_limit, cooldown_range=self.cooldown_range, verbose=self.settings.verbose) handler = create_rendezvous_handler(self.driver) logger.debug("[ray] starting rendezvous") global_rendezv_port = self.rendezvous.start(handler) logger.debug(f"[ray] waiting for {self.num_workers} to start.") self.driver.wait_for_available_slots(self.num_workers) # Host-to-host common interface detection # requires at least 2 hosts in an elastic job. min_hosts = _get_min_start_hosts(self.settings) current_hosts = self.driver.wait_for_available_slots( self.num_workers, min_hosts=min_hosts) logger.debug("[ray] getting common interfaces") nics = detect_nics( self.settings, all_host_names=current_hosts.host_assignment_order, ) logger.debug("[ray] getting driver IP") server_ip = socket.gethostbyname(socket.gethostname()) self.run_env_vars = create_run_env_vars(server_ip, nics, global_rendezv_port, elastic=True) self.executable_cls = executable_cls self.executable_args = executable_args self.executable_kwargs = executable_kwargs self.env_vars = extra_env_vars or {}
def elastic_driver_fn(): global_rendezv = RendezvousServer(verbose=1) discover_hosts = discovery.HostDiscoveryScript( "/Users/zuston/iqiyiDev/horovod-opal/dis.sh", 3) driver = ElasticDriver(global_rendezv, discover_hosts, min_np=2, max_np=4) handler = create_rendezvous_handler(driver) global_rendezv_port = global_rendezv.start(handler) print('port: ' + str(global_rendezv_port)) print('wait for available slots: {}'.format(2)) current_hosts = driver.wait_for_available_slots(2) print("current hosts:" + str(current_hosts)) pending_slots = driver._update_host_assignments(current_hosts) print("pending hosts:" + str(pending_slots)) driver._worker_registry.reset(driver.world_size())
def launch_gloo_elastic(command, exec_command, settings, env, get_common_interfaces, rendezvous): # Make the output directory if it does not exist if settings.output_filename: _mkdir_p(settings.output_filename) driver = ElasticDriver(rendezvous, settings.discovery, settings.min_np, settings.max_np, timeout=settings.elastic_timeout, reset_limit=settings.reset_limit, cooldown_range=settings.cooldown_range, verbose=settings.verbose) handler = create_rendezvous_handler(driver) global_rendezv_port = rendezvous.start(handler) driver.wait_for_available_slots(settings.num_proc) nics = get_common_interfaces(driver) server_ip = network.get_driver_ip(nics) event = register_shutdown_event() run_command = get_run_command(command, server_ip, nics, global_rendezv_port, elastic=True) create_worker = _create_elastic_worker_fn(exec_command, run_command, env, event) driver.start(settings.num_proc, create_worker) res = driver.get_results() driver.stop() if res.error_message is not None: raise RuntimeError(res.error_message) for name, value in sorted(res.worker_results.items(), key=lambda item: item[1][1]): exit_code, timestamp = value if exit_code != 0: raise RuntimeError( 'Horovod detected that one or more processes exited with non-zero ' 'status, thus causing the job to be terminated. The first process ' 'to do so was:\nProcess name: {name}\nExit code: {code}\n'. format(name=name, code=exit_code))
def launch_gloo_elastic(command_or_func, exec_command, settings, env, get_common_interfaces, rendezvous, executable): # Make the output directory if it does not exist if settings.output_filename: _mkdir_p(settings.output_filename) driver = ElasticDriver(rendezvous, settings.discovery, settings.min_num_proc, settings.max_num_proc, timeout=settings.elastic_timeout, reset_limit=settings.reset_limit, cooldown_range=settings.cooldown_range, verbose=settings.verbose) handler = create_rendezvous_handler(driver) global_rendezv_port = rendezvous.start(handler) driver.wait_for_available_slots(settings.num_proc) nics = get_common_interfaces(driver) server_ip = network.get_driver_ip(nics) run_func_server = None run_func_server_port = None if settings.run_func_mode: # when running a func, we have to spin up the KVStoreServer # to get the func to the remote process and the result back run_func_server = KVStoreServer(verbose=settings.verbose) run_func_server_port = run_func_server.start_server() put_data_into_kvstore(server_ip, run_func_server_port, 'runfunc', 'func', command_or_func) command = [executable, '-m', 'horovod.runner.run_task', server_ip, str(run_func_server_port)] else: command = command_or_func try: event = register_shutdown_event() run_command = get_run_command(command, server_ip, nics, global_rendezv_port, elastic=True) create_worker = _create_elastic_worker_fn(exec_command, run_command, env, event) driver.start(settings.num_proc, create_worker) res = driver.get_results() driver.stop() if res.error_message is not None: raise RuntimeError(res.error_message) for name, value in sorted(res.worker_results.items(), key=lambda item: item[1][1]): exit_code, timestamp = value if exit_code != 0: raise RuntimeError('Horovod detected that one or more processes exited with non-zero ' 'status, thus causing the job to be terminated. The first process ' 'to do so was:\nProcess name: {name}\nExit code: {code}\n' .format(name=name, code=exit_code)) # fetch the result if running a func if settings.run_func_mode: results = [None] * settings.min_num_proc # TODO: make it parallel to improve performance for i in range(settings.min_num_proc): results[i] = read_data_from_kvstore(server_ip, run_func_server_port, 'runfunc_result', str(i)) return results return None finally: if run_func_server: run_func_server.shutdown_server()
def test_worker_notification_manager(self): """Tests that host add events are sent to the worker notification service and consumed.""" slots = {'host-1': 2} discovery = FixedHosts(slots) rendezvous = RendezvousServer() driver = ElasticDriver(rendezvous, discovery, min_np=2, max_np=4) driver.wait_for_available_slots(min_np=2) handler = create_rendezvous_handler(driver) common_intfs = network.get_local_intfs() addr = network.get_driver_ip(common_intfs) port = rendezvous.start(handler) nic = list(common_intfs)[0] rank_results = {} class NotificationReceiver: def __init__(self): self.events = [] def on_hosts_updated(self, timestamp, res): self.events.append((timestamp, res)) def add_host(): slots = {'host-1': 2, 'host-2': 2} discovery.set(slots) def remove_host(): slots = {'host-2': 2} discovery.set(slots) def exec_command(slot_info, events): manager = WorkerNotificationManager() manager.init(rendezvous_addr=addr, rendezvous_port=port, nic=nic, hostname=slot_info.hostname, local_rank=slot_info.local_rank) notification_receiver = NotificationReceiver() manager.register_listener(notification_receiver) driver.record_ready(slot_info.hostname, slot_info.local_rank) if slot_info.rank == 0: add_host() driver.wait_for_available_slots(4) if slot_info.rank == 0: remove_host() # Busy wait for the number of available slots to decrease while driver._host_manager.current_hosts.count_available_slots( ) > 2: time.sleep(0.01) rank_results[slot_info.rank] = notification_receiver.events return 0, time.time() driver.start(np=2, create_worker_fn=exec_command) res = driver.get_results().worker_results driver.stop() assert len(res) == 2 for name, (exit_code, timestamp) in res.items(): assert exit_code == 0, name assert len(rank_results) == 2 for rank, events in rank_results.items(): expected = 2 if rank == 0 else 0 assert len(events) == expected, rank if rank == 0: # First update is an add assert events[0][1] == HostUpdateResult.added # Second update is a removal assert events[1][1] == HostUpdateResult.removed rendezvous.stop()