class HorovodBackend(Backend):
share_cuda_visible_devices: bool = True
def on_start(self, worker_group: WorkerGroup, backend_config: HorovodConfig):
# TODO(matt): Implement placement group strategies in BackendExecutor.
# Initialize workers with Horovod environment variables
setup_futures = []
for rank in range(len(worker_group)):
worker_node_id = worker_group.workers[rank].metadata.node_id
setup_futures.append(
worker_group.execute_single_async(
rank, init_env_vars, rank, len(worker_group), worker_node_id
)
)
ray.get(setup_futures)
# Use Horovod Ray Coordinator
# backend_config as settings
self.coordinator = Coordinator(backend_config)
# Get all the hostnames of all workers
node_ids = [w.metadata.node_id for w in worker_group.workers]
hostnames = [w.metadata.hostname for w in worker_group.workers]
# Register each hostname to the coordinator. assumes the hostname
# ordering is the same.
for rank, (hostname, node_id) in enumerate(zip(hostnames, node_ids)):
self.coordinator.register(hostname, node_id, rank)
all_info = self.coordinator.finalize_registration()
setup_futures = []
for rank, local_cross_env_var in all_info.items():
setup_futures.append(
worker_group.execute_single_async(
rank, update_env_vars, local_cross_env_var
)
)
ray.get(setup_futures)
coordinator_envs = self.coordinator.establish_rendezvous()
# Get one worker from each host/node.
node_worker_indexes = [node_ids.index(node_id) for node_id in set(node_ids)]
node_workers = [
HorovodWorkerWrapper(worker_group.workers[worker_index])
for worker_index in node_worker_indexes
]
assert len(node_workers) == len(self.coordinator.hostnames)
nics = detect_nics(
backend_config,
all_host_names=list(self.coordinator.hostnames),
node_workers=node_workers,
)
coordinator_envs.update(nics_to_env_var(nics))
worker_group.execute(update_env_vars, coordinator_envs)
class HorovodBackend(Backend):
def on_start(self, worker_group: WorkerGroup,
backend_config: HorovodConfig):
# TODO(matt): Implement placement group strategies in BackendExecutor.
# Initialize workers with Horovod environment variables
setup_futures = []
for rank in range(len(worker_group)):
setup_futures.append(
worker_group.execute_single_async(rank, init_env_vars, rank,
len(worker_group)))
ray.get(setup_futures)
# Use Horovod Ray Coordinator
# backend_config as settings
self.coordinator = Coordinator(backend_config)
# Get all the hostnames of all workers
node_ids = worker_group.execute(get_node_id)
hostnames = worker_group.execute(get_hostname)
# Register each hostname to the coordinator. assumes the hostname
# ordering is the same.
for rank, (hostname, node_id) in enumerate(zip(hostnames, node_ids)):
self.coordinator.register(hostname, node_id, rank)
all_info = self.coordinator.finalize_registration()
setup_futures = []
for rank, local_cross_env_var in all_info.items():
setup_futures.append(
worker_group.execute_single_async(rank, update_env_vars,
local_cross_env_var))
ray.get(setup_futures)
coordinator_envs = self.coordinator.establish_rendezvous()
nics = detect_nics(backend_config,
all_host_names=list(self.coordinator.hostnames),
node_workers=worker_group.workers)
coordinator_envs.update(nics_to_env_var(nics))
worker_group.execute(update_env_vars, coordinator_envs)
