def start(self):
"""Starts the Horovod driver and services."""
self.rendezvous = RendezvousServer(self.settings.verbose)
self.driver = ElasticDriver(rendezvous=self.rendezvous,
discovery=self.settings.discovery,
min_np=self.settings.min_np,
max_np=self.settings.max_np,
timeout=self.settings.elastic_timeout,
reset_limit=self.settings.reset_limit,
verbose=self.settings.verbose)
handler = create_rendezvous_handler(self.driver)
global_rendezv_port = self.rendezvous.start(handler)
self.driver.wait_for_available_slots(self.settings.num_proc)
# Host-to-host common interface detection
# requires at least 2 hosts in an elastic job.
min_hosts = _get_min_start_hosts(self.settings)
current_hosts = self.driver.wait_for_available_slots(
self.settings.num_proc, min_hosts=min_hosts)
nics = driver_service.get_common_interfaces(
self.settings, current_hosts.host_assignment_order)
server_ip = network.get_driver_ip(nics)
self.run_env_vars = create_run_env_vars(server_ip,
nics,
global_rendezv_port,
elastic=True)
def start(self):
"""Starts the Horovod driver and services."""
self.rendezvous = RendezvousServer(self.settings.verbose)
self.driver = ElasticDriver(rendezvous=self.rendezvous,
discovery=self.settings.discovery,
min_num_proc=self.settings.min_num_proc,
max_num_proc=self.settings.max_num_proc,
timeout=self.settings.elastic_timeout,
reset_limit=self.settings.reset_limit,
verbose=self.settings.verbose)
handler = create_rendezvous_handler(self.driver)
logger.debug("[ray] starting rendezvous")
global_rendezv_port = self.rendezvous.start(handler)
logger.debug(f"[ray] waiting for {self.settings.num_proc} to start.")
self.driver.wait_for_available_slots(self.settings.num_proc)
# Host-to-host common interface detection
# requires at least 2 hosts in an elastic job.
min_hosts = _get_min_start_hosts(self.settings)
current_hosts = self.driver.wait_for_available_slots(
self.settings.num_proc, min_hosts=min_hosts)
logger.debug("[ray] getting common interfaces")
nics = detect_nics(
self.settings,
all_host_names=current_hosts.host_assignment_order,
)
logger.debug("[ray] getting driver IP")
server_ip = socket.gethostbyname(socket.gethostname())
self.run_env_vars = create_run_env_vars(server_ip,
nics,
global_rendezv_port,
elastic=True)
def start(self,
executable_cls: type = None,
executable_args: Optional[List] = None,
executable_kwargs: Optional[Dict] = None,
extra_env_vars: Optional[Dict] = None):
"""Starts the Horovod driver and services.
Args:
executable_cls (type): The class that will be created within
an actor (BaseHorovodWorker). This will allow Horovod
to establish its connections and set env vars.
executable_args (List): Arguments to be passed into the
worker class upon initialization.
executable_kwargs (Dict): Keyword arguments to be passed into the
worker class upon initialization.
extra_env_vars (Dict): Environment variables to be set
on the actors (worker processes) before initialization.
"""
self.rendezvous = RendezvousServer(self.settings.verbose)
self.driver = ElasticDriver(rendezvous=self.rendezvous,
discovery=self.settings.discovery,
min_np=self.min_workers,
max_np=self.max_workers,
timeout=self.elastic_timeout,
reset_limit=self.reset_limit,
cooldown_range=self.cooldown_range,
verbose=self.settings.verbose)
handler = create_rendezvous_handler(self.driver)
logger.debug("[ray] starting rendezvous")
global_rendezv_port = self.rendezvous.start(handler)
logger.debug(f"[ray] waiting for {self.num_workers} to start.")
self.driver.wait_for_available_slots(self.num_workers)
# Host-to-host common interface detection
# requires at least 2 hosts in an elastic job.
min_hosts = _get_min_start_hosts(self.settings)
current_hosts = self.driver.wait_for_available_slots(
self.num_workers, min_hosts=min_hosts)
logger.debug("[ray] getting common interfaces")
nics = detect_nics(
self.settings,
all_host_names=current_hosts.host_assignment_order,
)
logger.debug("[ray] getting driver IP")
server_ip = socket.gethostbyname(socket.gethostname())
self.run_env_vars = create_run_env_vars(server_ip,
nics,
global_rendezv_port,
elastic=True)
self.executable_cls = executable_cls
self.executable_args = executable_args
self.executable_kwargs = executable_kwargs
self.env_vars = extra_env_vars or {}
def test_rank_and_size(self):
"""Tests two hosts, two slots each in standard happy path."""
slots = {'host-1': 2, 'host-2': 2}
discovery = FixedHosts(slots)
driver = ElasticDriver(mock.Mock(), discovery, min_np=2, max_np=4)
driver.wait_for_available_slots(min_np=2)
rank_results = {}
def exec_command(slot_info, events):
driver.record_ready(slot_info.hostname, slot_info.local_rank)
updated_slot_info = driver.get_slot_info(slot_info.hostname,
slot_info.local_rank)
rank_results[slot_info.rank] = (slot_info, updated_slot_info)
return 0, time.time()
driver.start(np=2, create_worker_fn=exec_command)
res = driver.get_results().worker_results
driver.stop()
assert len(res) == 4
for name, (exit_code, timestamp) in res.items():
assert exit_code == 0, name
assert len(rank_results) == 4
for rank, (slot_info, updated_slot_info) in rank_results.items():
assert slot_info.to_response_string(
) == updated_slot_info.to_response_string(), rank
def test_rank_and_size_with_host_failure(self):
"""Tests two hosts, two slots each with second host failing before rendezvous completes."""
slots = {'host-1': 2, 'host-2': 2}
discovery = FixedHosts(slots)
driver = ElasticDriver(mock.Mock(), discovery, min_num_proc=2, max_num_proc=4)
driver.wait_for_available_slots(min_num_proc=2)
rank_results = {}
def exec_command(slot_info, events):
if slot_info.hostname == 'host-2':
return 1, time.time()
driver.record_ready(slot_info.hostname, slot_info.local_rank)
updated_slot_info = driver.get_slot_info(slot_info.hostname, slot_info.local_rank)
rank_results[slot_info.rank] = (slot_info, updated_slot_info)
return 0, time.time()
driver.start(num_proc=2, create_worker_fn=exec_command)
res = driver.get_results().worker_results
driver.stop()
assert len(res) == 2
for name, (exit_code, timestamp) in res.items():
assert exit_code == 0, name
assert len(rank_results) == 2
for rank, (slot_info, updated_slot_info) in rank_results.items():
assert updated_slot_info.size == 2, rank
assert updated_slot_info.rank == slot_info.rank % 2, rank
assert updated_slot_info.local_size == slot_info.local_size, rank
assert updated_slot_info.local_rank == slot_info.local_rank, rank
assert updated_slot_info.cross_size == 1, rank
assert updated_slot_info.cross_rank == 0, rank
def test_shutdown_on_success(self):
"""Tests that shutdown event is triggered when one worker succeeds but the others are still working."""
slots = {'host-1': 2, 'host-2': 2}
discovery = FixedHosts(slots)
driver = ElasticDriver(mock.Mock(), discovery, min_np=2, max_np=4)
driver.wait_for_available_slots(min_np=2)
def exec_command(slot_info, events):
if slot_info.rank == 0:
return 0, time.time()
driver.record_ready(slot_info.hostname, slot_info.local_rank)
wait_for_one(events)
return 1, time.time()
driver.start(np=2, create_worker_fn=exec_command)
res = driver.get_results().worker_results
driver.stop()
assert len(res) == 4
exit_code_sum = 0
for name, (exit_code, timestamp) in res.items():
exit_code_sum += exit_code
assert exit_code_sum == 3
def elastic_driver_fn():
global_rendezv = RendezvousServer(verbose=1)
discover_hosts = discovery.HostDiscoveryScript(
"/Users/zuston/iqiyiDev/horovod-opal/dis.sh", 3)
driver = ElasticDriver(global_rendezv, discover_hosts, min_np=2, max_np=4)
handler = create_rendezvous_handler(driver)
global_rendezv_port = global_rendezv.start(handler)
print('port: ' + str(global_rendezv_port))
print('wait for available slots: {}'.format(2))
current_hosts = driver.wait_for_available_slots(2)
print("current hosts:" + str(current_hosts))
pending_slots = driver._update_host_assignments(current_hosts)
print("pending hosts:" + str(pending_slots))
driver._worker_registry.reset(driver.world_size())
def launch_gloo_elastic(command, exec_command, settings, env,
get_common_interfaces, rendezvous):
# Make the output directory if it does not exist
if settings.output_filename:
_mkdir_p(settings.output_filename)
driver = ElasticDriver(rendezvous,
settings.discovery,
settings.min_np,
settings.max_np,
timeout=settings.elastic_timeout,
reset_limit=settings.reset_limit,
cooldown_range=settings.cooldown_range,
verbose=settings.verbose)
handler = create_rendezvous_handler(driver)
global_rendezv_port = rendezvous.start(handler)
driver.wait_for_available_slots(settings.num_proc)
nics = get_common_interfaces(driver)
server_ip = network.get_driver_ip(nics)
event = register_shutdown_event()
run_command = get_run_command(command,
server_ip,
nics,
global_rendezv_port,
elastic=True)
create_worker = _create_elastic_worker_fn(exec_command, run_command, env,
event)
driver.start(settings.num_proc, create_worker)
res = driver.get_results()
driver.stop()
if res.error_message is not None:
raise RuntimeError(res.error_message)
for name, value in sorted(res.worker_results.items(),
key=lambda item: item[1][1]):
exit_code, timestamp = value
if exit_code != 0:
raise RuntimeError(
'Horovod detected that one or more processes exited with non-zero '
'status, thus causing the job to be terminated. The first process '
'to do so was:\nProcess name: {name}\nExit code: {code}\n'.
format(name=name, code=exit_code))
def test_rank_and_size_with_host_added(self):
"""Tests training starts with one host two slots, then a second host is added."""
slots = {'host-1': 2}
discovery = FixedHosts(slots)
def add_host():
slots = {'host-1': 2, 'host-2': 2}
discovery.set(slots)
driver = ElasticDriver(mock.Mock(), discovery, min_np=2, max_np=4)
driver.wait_for_available_slots(min_np=2)
rank_results = {}
def exec_command(slot_info, events):
driver.record_ready(slot_info.hostname, slot_info.local_rank)
if slot_info.hostname == 'host-1':
if slot_info.rank == 0:
add_host()
driver.wait_for_available_slots(4)
driver.record_ready(slot_info.hostname, slot_info.local_rank)
driver.record_ready(slot_info.hostname, slot_info.local_rank)
updated_slot_info = driver.get_slot_info(slot_info.hostname,
slot_info.local_rank)
rank_results[slot_info.rank] = (slot_info, updated_slot_info)
return 0, time.time()
driver.start(np=2, create_worker_fn=exec_command)
res = driver.get_results().worker_results
driver.stop()
assert len(res) == 4
for name, (exit_code, timestamp) in res.items():
assert exit_code == 0, name
assert len(rank_results) == 4
for rank, (slot_info, updated_slot_info) in rank_results.items():
assert updated_slot_info.size == 4, rank
assert updated_slot_info.rank == slot_info.rank, rank
assert updated_slot_info.local_size == slot_info.local_size, rank
assert updated_slot_info.local_rank == slot_info.local_rank, rank
assert updated_slot_info.cross_size == 2, rank
assert updated_slot_info.cross_rank == slot_info.cross_rank, rank
def test_wait_for_available_slots(self, mock_get_worker_client,
mock_get_coordinator_info):
"""Tests that driver blocks until the min number of slots are available."""
slots = [{
'host-1': 4
}, {
'host-1': 4,
'host-2': 8
}, {
'host-1': 4,
'host-2': 8,
'host-3': 4
}]
mock_discovery = mock.Mock()
mock_discovery.find_available_hosts_and_slots.side_effect = sequence(
slots)
driver = ElasticDriver(mock.Mock(),
mock_discovery,
min_np=8,
max_np=20)
driver.wait_for_available_slots(min_np=16)
assert driver._host_manager.current_hosts.count_available_slots() >= 16
driver.stop()
# Notify coordinator 2 times, as the first time we are below min_np and the existing host assignments
# are empty
assert mock_get_worker_client.call_count == 2
assert mock_get_coordinator_info.call_count == 2
def test_send_notifications_without_assignments(self,
mock_get_worker_client,
mock_get_coordinator_info,
mock_host_assignments):
"""Tests that notifications are still sent correctly even if host assignments cannot be generated."""
slots = [{
'host-1': 8,
'host-2': 4
}, {
'host-1': 8,
'host-2': 4
}, {
'host-2': 4
}, {
'host-2': 4
}, {
'host-2': 4,
'host-3': 12
}]
discovery = mock.Mock()
discovery.find_available_hosts_and_slots.side_effect = sequence(slots)
driver = ElasticDriver(mock.Mock(), discovery, min_np=8, max_np=12)
driver.wait_for_available_slots(min_np=16)
driver.stop()
# On the second call, we should see the number of slots dip below the minimum, but we still want to ensure
# we notify workers every time there is a change, so in total we should observe 3 calls.
assert mock_get_worker_client.call_count == 3
assert mock_get_coordinator_info.call_count == 3
def test_wait_for_min_hosts(self):
"""Tests that driver blocks until the min number of hosts and slots are available."""
slots = [{
'host-1': 4
}, {
'host-1': 4,
'host-2': 8
}, {
'host-1': 4,
'host-2': 8,
'host-3': 4
}]
mock_discovery = mock.Mock()
mock_discovery.find_available_hosts_and_slots.side_effect = sequence(
slots)
driver = ElasticDriver(mock.Mock(),
mock_discovery,
min_np=2,
max_np=12)
driver.wait_for_available_slots(min_np=2, min_hosts=2)
# Even though we only needed 2 slots, because we also needed 2 hosts, we will at least 12 slots total
assert driver._host_manager.current_hosts.count_available_slots() >= 12
driver.stop()
def test_shutdown_on_initial_discovery_failure(self):
"""Tests that the driver will shutdown immediately if initial host discovery fails."""
discovery = mock.Mock()
discovery.find_available_hosts_and_slots.side_effect = RuntimeError()
discover_hosts = ElasticDriver._discover_hosts
def wrapped_discover_hosts(obj):
try:
discover_hosts(obj)
except RuntimeError:
# Suppress the error message from the background discovery thread to clean up unit tests
pass
try:
ElasticDriver._discover_hosts = wrapped_discover_hosts
driver = ElasticDriver(mock.Mock(), discovery, min_np=2, max_np=4)
with pytest.raises(RuntimeError):
driver.wait_for_available_slots(min_np=2)
assert driver.finished()
finally:
ElasticDriver._discover_hosts = discover_hosts
def test_all_workers_fail(self):
"""Tests that training fails when all workers fail."""
slots = {'host-1': 2, 'host-2': 2}
discovery = FixedHosts(slots)
driver = ElasticDriver(mock.Mock(), discovery, min_np=2, max_np=4)
driver.wait_for_available_slots(min_np=2)
def exec_command(slot_info, events):
driver.record_ready(slot_info.hostname, slot_info.local_rank)
return 1, time.time()
driver.start(np=2, create_worker_fn=exec_command)
res = driver.get_results().worker_results
driver.stop()
assert len(res) == 4
for name, (exit_code, timestamp) in res.items():
assert exit_code == 1, name
class ElasticRayExecutor:
"""Executor for elastic jobs using Ray.
Leverages the Ray global state to detect available hosts and
slots. Assumes that the entire Ray cluster is available for
the Executor to use.
Args:
settings: Configuration for the elastic job
setup. You can use a standard Horovod ElasticSettings
object or create one directly from
ElasticRayExecutor.create_settings.
use_gpu (bool): Whether to use GPU for allocation.
cpus_per_slot (int): Number of CPU resources to allocate to
each worker.
gpus_per_slot (int): Number of GPU resources to allocate to
each worker.
env_vars (Dict): Environment variables to be set
on the actors (worker processes) before initialization.
override_discovery (bool): Whether for the ElasticRayExecutor to
automatically provide a discovery mechanism for ElasticSettings.
Example:
.. code-block:: python
import ray
ray.init(address="auto")
settings = ElasticRayExecutor.create_settings(verbose=True)
executor = ElasticRayExecutor(
settings, use_gpu=True, cpus_per_slot=2)
executor.start()
executor.run(train_fn)
warning:: .. deprecated:: 0.25.0
"""
@staticmethod
def create_settings(
min_num_proc: int = 1,
max_num_proc: int = None,
reset_limit: int = None,
elastic_timeout: int = 600,
timeout_s: int = 30,
ssh_identity_file: str = None,
nics: str = None,
# min_np is deprecated, use min_num_proc instead
min_np=None,
# max_np is deprecated, use max_num_proc instead
max_np=None,
**kwargs):
"""Returns a Settings object for ElasticRayExecutor.
Note that the `discovery` property will be set at runtime.
Args:
min_num_proc (int): Minimum number of processes running for
training to continue. If number of available processes dips
below this threshold, then training will wait for
more instances to become available.
max_num_proc (int): Maximum number of training processes,
beyond which no additional processes will be created.
If not specified, then will be unbounded.
reset_limit (int): Maximum number of times that the training
job can scale up or down the number of workers after
which the job is terminated.
elastic_timeout (int): Timeout for elastic initialisation after
re-scaling the cluster. The default value is 600 seconds.
Alternatively, the environment variable
HOROVOD_ELASTIC_TIMEOUT can also be used.'
timeout_s (int): Horovod performs all the checks and starts the
processes before the specified timeout.
The default value is 30 seconds.
ssh_identity_file (str): File on the driver from which
the identity (private key) is read.
nics (set): Network interfaces that can be used for communication.
"""
if min_np is not None:
min_num_proc = min_np
warnings.warn('min_np is deprecated, use min_num_proc instead',
DeprecationWarning)
if max_np is not None:
max_num_proc = max_np
warnings.warn('max_np is deprecated, use max_num_proc instead',
DeprecationWarning)
start_timeout = timeout.Timeout(
timeout_s,
message="Timed out waiting for {activity}. Please "
"check connectivity between servers. You "
"may need to increase the --start-timeout "
"parameter if you have too many servers.")
ssh_identity_file = ssh_identity_file or os.path.expanduser(
"~/ray_bootstrap_key.pem")
settings = ElasticSettings(
discovery=None,
min_num_proc=min_num_proc,
max_num_proc=max_num_proc,
elastic_timeout=elastic_timeout,
reset_limit=reset_limit,
num_proc=min_num_proc,
ssh_identity_file=ssh_identity_file,
nics=nics,
start_timeout=start_timeout,
key=secret.make_secret_key() if secret else None,
**kwargs)
return settings
def __init__(self,
settings: ElasticSettings,
use_gpu: bool = False,
cpus_per_slot: int = 1,
gpus_per_slot: Optional[int] = None,
env_vars: dict = None,
override_discovery=True):
if gpus_per_slot and not use_gpu:
raise ValueError("gpus_per_slot is set, but use_gpu is False. "
"use_gpu must be True if gpus_per_slot is set. ")
gpus_per_slot = gpus_per_slot or int(use_gpu)
if use_gpu and gpus_per_slot < 1:
raise ValueError(
f"gpus_per_slot must be >= 1: Got {gpus_per_slot}.")
if override_discovery:
settings.discovery = RayHostDiscovery(use_gpu=use_gpu,
cpus_per_slot=cpus_per_slot,
gpus_per_slot=gpus_per_slot)
self.cpus_per_slot = cpus_per_slot
self.gpus_per_slot = gpus_per_slot
self.use_gpu = use_gpu
self.settings = settings
self.driver = None
self.rendezvous = None
self.env_vars = env_vars or {}
def start(self):
"""Starts the Horovod driver and services."""
self.rendezvous = RendezvousServer(self.settings.verbose)
self.driver = ElasticDriver(rendezvous=self.rendezvous,
discovery=self.settings.discovery,
min_num_proc=self.settings.min_num_proc,
max_num_proc=self.settings.max_num_proc,
timeout=self.settings.elastic_timeout,
reset_limit=self.settings.reset_limit,
verbose=self.settings.verbose)
handler = create_rendezvous_handler(self.driver)
logger.debug("[ray] starting rendezvous")
global_rendezv_port = self.rendezvous.start(handler)
logger.debug(f"[ray] waiting for {self.settings.num_proc} to start.")
self.driver.wait_for_available_slots(self.settings.num_proc)
# Host-to-host common interface detection
# requires at least 2 hosts in an elastic job.
min_hosts = _get_min_start_hosts(self.settings)
current_hosts = self.driver.wait_for_available_slots(
self.settings.num_proc, min_hosts=min_hosts)
logger.debug("[ray] getting common interfaces")
nics = detect_nics(
self.settings,
all_host_names=current_hosts.host_assignment_order,
)
logger.debug("[ray] getting driver IP")
server_ip = socket.gethostbyname(socket.gethostname())
self.run_env_vars = create_run_env_vars(server_ip,
nics,
global_rendezv_port,
elastic=True)
def _create_resources(self, hostname: str):
resources = dict(num_cpus=self.cpus_per_slot,
num_gpus=int(self.use_gpu) * self.gpus_per_slot,
resources={f"node:{hostname}": 0.01})
return resources
def _create_remote_worker(self, slot_info, worker_env_vars):
hostname = slot_info.hostname
loaded_worker_cls = self.remote_worker_cls.options(
**self._create_resources(hostname))
worker = loaded_worker_cls.remote()
worker.update_env_vars.remote(worker_env_vars)
worker.update_env_vars.remote(create_slot_env_vars(slot_info))
if self.use_gpu:
visible_devices = ",".join(
[str(i) for i in range(slot_info.local_size)])
worker.update_env_vars.remote(
{"CUDA_VISIBLE_DEVICES": visible_devices})
return worker
def _create_spawn_worker_fn(self, return_results: List,
worker_fn: Callable,
queue: "ray.util.Queue") -> Callable:
self.remote_worker_cls = ray.remote(BaseHorovodWorker)
# event = register_shutdown_event()
worker_env_vars = {}
worker_env_vars.update(self.run_env_vars.copy())
worker_env_vars.update(self.env_vars.copy())
worker_env_vars.update({"PYTHONUNBUFFERED": "1"})
def worker_loop(slot_info, events):
def ping_worker(worker):
# There is an odd edge case where a node can be removed
# before the remote worker is started, leading to a failure
# in trying to create the horovod mesh.
try:
ping = worker.execute.remote(lambda _: 1)
ray.get(ping, timeout=10)
except Exception as e:
logger.error(f"{slot_info.hostname}: Ping failed - {e}")
return False
return True
worker = self._create_remote_worker(slot_info, worker_env_vars)
if not ping_worker(worker):
return 1, time.time()
ray.get(worker.set_queue.remote(queue))
future = worker.execute.remote(lambda _: worker_fn())
result = None
while result is None:
try:
# TODO: make this event driven at some point.
retval = ray.get(future, timeout=0.1)
return_results.append((slot_info.rank, retval))
# Success
result = 0, time.time()
except GetTimeoutError:
# Timeout
if any(e.is_set() for e in events):
ray.kill(worker)
result = 1, time.time()
except Exception as e:
logger.error(f"{slot_info.hostname}[{slot_info.rank}]:{e}")
ray.kill(worker)
result = 1, time.time()
logger.debug(f"Worker ({slot_info}) routine is done!")
return result
return worker_loop
def run(self,
worker_fn: Callable,
callbacks: Optional[List[Callable]] = None) -> List[Any]:
"""Executes the provided function on all workers.
Args:
worker_fn: Target elastic function that can be executed.
callbacks: List of callables. Each callback must either
be a callable function or a class that implements __call__.
Every callback will be invoked on every value logged
by the rank 0 worker.
Returns:
List of return values from every completed worker.
"""
return_values = []
from ray.util.queue import Queue
import inspect
args = inspect.getfullargspec(Queue).args
if "actor_options" not in args:
# Ray 1.1 and less
_queue = Queue()
else:
_queue = Queue(actor_options={
"num_cpus": 0,
"resources": {
ray.state.current_node_id(): 0.001
}
})
self.driver.start(
self.settings.num_proc,
self._create_spawn_worker_fn(return_values, worker_fn, _queue))
def _process_calls(queue, callbacks, event):
if not callbacks:
return
while queue.actor:
if not queue.empty():
result = queue.get_nowait()
for c in callbacks:
c(result)
# avoid slamming the CI
elif event.is_set():
break
time.sleep(0.1)
try:
event = threading.Event()
_callback_thread = threading.Thread(target=_process_calls,
args=(_queue, callbacks,
event),
daemon=True)
_callback_thread.start()
res = self.driver.get_results()
event.set()
if _callback_thread:
_callback_thread.join(timeout=60)
finally:
if hasattr(_queue, "shutdown"):
_queue.shutdown()
else:
done_ref = _queue.actor.__ray_terminate__.remote()
done, not_done = ray.wait([done_ref], timeout=5)
if not_done:
ray.kill(_queue.actor)
self.driver.stop()
if res.error_message is not None:
raise RuntimeError(res.error_message)
for name, value in sorted(res.worker_results.items(),
key=lambda item: item[1][1]):
exit_code, timestamp = value
if exit_code != 0:
raise RuntimeError(
'Horovod detected that one or more processes '
'exited with non-zero '
'status, thus causing the job to be terminated. '
'The first process '
'to do so was:\nProcess name: {name}\nExit code: {code}\n'.
format(name=name, code=exit_code))
return_values = [
value for k, value in sorted(return_values, key=lambda kv: kv[0])
]
return return_values
def launch_gloo_elastic(command_or_func, exec_command, settings, env, get_common_interfaces, rendezvous, executable):
# Make the output directory if it does not exist
if settings.output_filename:
_mkdir_p(settings.output_filename)
driver = ElasticDriver(rendezvous, settings.discovery,
settings.min_num_proc, settings.max_num_proc,
timeout=settings.elastic_timeout,
reset_limit=settings.reset_limit,
cooldown_range=settings.cooldown_range,
verbose=settings.verbose)
handler = create_rendezvous_handler(driver)
global_rendezv_port = rendezvous.start(handler)
driver.wait_for_available_slots(settings.num_proc)
nics = get_common_interfaces(driver)
server_ip = network.get_driver_ip(nics)
run_func_server = None
run_func_server_port = None
if settings.run_func_mode:
# when running a func, we have to spin up the KVStoreServer
# to get the func to the remote process and the result back
run_func_server = KVStoreServer(verbose=settings.verbose)
run_func_server_port = run_func_server.start_server()
put_data_into_kvstore(server_ip, run_func_server_port, 'runfunc', 'func', command_or_func)
command = [executable, '-m', 'horovod.runner.run_task', server_ip, str(run_func_server_port)]
else:
command = command_or_func
try:
event = register_shutdown_event()
run_command = get_run_command(command, server_ip, nics, global_rendezv_port, elastic=True)
create_worker = _create_elastic_worker_fn(exec_command, run_command, env, event)
driver.start(settings.num_proc, create_worker)
res = driver.get_results()
driver.stop()
if res.error_message is not None:
raise RuntimeError(res.error_message)
for name, value in sorted(res.worker_results.items(), key=lambda item: item[1][1]):
exit_code, timestamp = value
if exit_code != 0:
raise RuntimeError('Horovod detected that one or more processes exited with non-zero '
'status, thus causing the job to be terminated. The first process '
'to do so was:\nProcess name: {name}\nExit code: {code}\n'
.format(name=name, code=exit_code))
# fetch the result if running a func
if settings.run_func_mode:
results = [None] * settings.min_num_proc
# TODO: make it parallel to improve performance
for i in range(settings.min_num_proc):
results[i] = read_data_from_kvstore(server_ip, run_func_server_port, 'runfunc_result', str(i))
return results
return None
finally:
if run_func_server:
run_func_server.shutdown_server()
class ElasticRayExecutor:
"""Executor for elastic jobs using Ray.
Leverages the Ray global state to detect available hosts and
slots. Assumes that the entire Ray cluster is available for
the Executor to use.
Args:
settings: Configuration for the elastic job
setup. You can use a standard Horovod ElasticSettings
object or create one directly from
ElasticRayExecutor.create_settings.
use_gpu (bool): Whether to use GPU for allocation.
cpus_per_slot (int): Number of CPU resources to allocate to
each worker.
gpus_per_slot (int): Number of GPU resources to allocate to
each worker.
env_vars (Dict): Environment variables to be set
on the actors (worker processes) before initialization.
override_discovery (bool): Whether for the ElasticRayExecutor to
automatically provide a discovery mechanism for ElasticSettings.
Example:
.. code-block:: python
import ray
ray.init(address="auto")
settings = ElasticRayExecutor.create_settings(verbose=True)
executor = ElasticRayExecutor(
settings, use_gpu=True, cpus_per_slot=2)
executor.start()
executor.run(train_fn)
"""
@staticmethod
def create_settings(min_np: int = 1,
max_np: int = None,
reset_limit: int = None,
elastic_timeout: int = 600,
timeout_s: int = 30,
ssh_identity_file: str = None,
nics: str = None,
**kwargs):
"""Returns a Settings object for ElasticRayExecutor.
Note that the `discovery` property will be set at runtime.
Args:
min_np (int): Minimum number of processes running for
training to continue. If number of available processes dips
below this threshold, then training will wait for
more instances to become available.
max_np (int): Maximum number of training processes,
beyond which no additional processes will be created.
If not specified, then will be unbounded.
reset_limit (int): Maximum number of times that the training
job can scale up or down the number of workers after
which the job is terminated.
elastic_timeout (int): Timeout for elastic initialisation after
re-scaling the cluster. The default value is 600 seconds.
Alternatively, the environment variable
HOROVOD_ELASTIC_TIMEOUT can also be used.'
timeout_s (int): Horovod performs all the checks and starts the
processes before the specified timeout.
The default value is 30 seconds.
ssh_identity_file (str): File on the driver from which
the identity (private key) is read.
nics (set): Network interfaces that can be used for communication.
"""
start_timeout = timeout.Timeout(
timeout_s,
message="Timed out waiting for {activity}. Please "
"check connectivity between servers. You "
"may need to increase the --start-timeout "
"parameter if you have too many servers.")
ssh_identity_file = ssh_identity_file or os.path.expanduser(
"~/ray_bootstrap_key.pem")
settings = ElasticSettings(
discovery=None,
min_np=min_np,
max_np=max_np,
elastic_timeout=elastic_timeout,
reset_limit=reset_limit,
num_proc=min_np,
ssh_identity_file=ssh_identity_file,
nics=nics,
start_timeout=start_timeout,
key=secret.make_secret_key() if secret else None,
**kwargs)
return settings
def __init__(self,
settings: ElasticSettings,
use_gpu: bool = False,
cpus_per_slot: int = 1,
gpus_per_slot: Optional[int] = None,
env_vars: dict = None,
override_discovery=True):
if gpus_per_slot and not use_gpu:
raise ValueError("gpus_per_slot is set, but use_gpu is False. "
"use_gpu must be True if gpus_per_slot is set. ")
if use_gpu and gpus_per_slot < 1:
raise ValueError(
f"gpus_per_slot must be >= 1: Got {gpus_per_slot}.")
gpus_per_slot = gpus_per_slot or 1
if override_discovery:
settings.discovery = RayHostDiscovery(use_gpu=use_gpu,
cpus_per_slot=cpus_per_slot,
gpus_per_slot=gpus_per_slot)
self.cpus_per_slot = cpus_per_slot
self.gpus_per_slot = gpus_per_slot
self.use_gpu = use_gpu
self.settings = settings
self.driver = None
self.rendezvous = None
self.env_vars = env_vars or {}
def start(self):
"""Starts the Horovod driver and services."""
self.rendezvous = RendezvousServer(self.settings.verbose)
self.driver = ElasticDriver(rendezvous=self.rendezvous,
discovery=self.settings.discovery,
min_np=self.settings.min_np,
max_np=self.settings.max_np,
timeout=self.settings.elastic_timeout,
reset_limit=self.settings.reset_limit,
verbose=self.settings.verbose)
handler = create_rendezvous_handler(self.driver)
global_rendezv_port = self.rendezvous.start(handler)
self.driver.wait_for_available_slots(self.settings.num_proc)
# Host-to-host common interface detection
# requires at least 2 hosts in an elastic job.
min_hosts = _get_min_start_hosts(self.settings)
current_hosts = self.driver.wait_for_available_slots(
self.settings.num_proc, min_hosts=min_hosts)
nics = driver_service.get_common_interfaces(
self.settings, current_hosts.host_assignment_order)
server_ip = network.get_driver_ip(nics)
self.run_env_vars = create_run_env_vars(server_ip,
nics,
global_rendezv_port,
elastic=True)
def _create_resources(self, hostname: str):
resources = dict(num_cpus=self.cpus_per_slot,
num_gpus=int(self.use_gpu) * self.gpus_per_slot,
resources={f"node:{hostname}": 0.01})
return resources
def _create_remote_worker(self, slot_info, worker_env_vars):
hostname = slot_info.hostname
loaded_worker_cls = self.remote_worker_cls.options(
**self._create_resources(hostname))
worker = loaded_worker_cls.remote()
worker.update_env_vars.remote(worker_env_vars)
worker.update_env_vars.remote(create_slot_env_vars(slot_info))
if self.use_gpu:
visible_devices = ",".join(
[str(i) for i in range(slot_info.local_size)])
worker.update_env_vars.remote(
{"CUDA_VISIBLE_DEVICES": visible_devices})
return worker
def _create_spawn_worker_fn(self, return_results: List,
worker_fn: Callable) -> Callable:
self.remote_worker_cls = ray.remote(BaseHorovodWorker)
# event = register_shutdown_event()
worker_env_vars = {}
worker_env_vars.update(self.run_env_vars.copy())
worker_env_vars.update(self.env_vars.copy())
worker_env_vars.update({"PYTHONUNBUFFERED": "1"})
def worker_loop(slot_info, events):
worker = self._create_remote_worker(slot_info, worker_env_vars)
future = worker.execute.remote(lambda _: worker_fn())
result = None
while result is None:
try:
# TODO: make this event driven at some point.
retval = ray.get(future, timeout=0.1)
return_results.append((slot_info.rank, retval))
# Success
result = 0, time.time()
except GetTimeoutError:
# Timeout
if any(e.is_set() for e in events):
ray.kill(worker)
result = 1, time.time()
except Exception as e:
logger.exception(str(e))
# Fail
result = 1, time.time()
return result
return worker_loop
def run(self, worker_fn: Callable) -> List[Any]:
"""Executes the provided function on all workers.
Args:
worker_fn: Target elastic function that can be executed.
Returns:
List of return values from every completed worker.
"""
return_values = []
self.driver.start(
self.settings.num_proc,
self._create_spawn_worker_fn(return_values, worker_fn))
res = self.driver.get_results()
self.driver.stop()
if res.error_message is not None:
raise RuntimeError(res.error_message)
for name, value in sorted(res.worker_results.items(),
key=lambda item: item[1][1]):
exit_code, timestamp = value
if exit_code != 0:
raise RuntimeError(
'Horovod detected that one or more processes '
'exited with non-zero '
'status, thus causing the job to be terminated. '
'The first process '
'to do so was:\nProcess name: {name}\nExit code: {code}\n'.
format(name=name, code=exit_code))
return_values = [
value for k, value in sorted(return_values, key=lambda kv: kv[0])
]
return return_values
class ElasticAdapter(Adapter):
"""Adapter for executing Ray calls for elastic Horovod jobs.
Args:
settings (horovod.Settings): Configuration for job setup. You can
use a standard Horovod Settings object or create one directly
from RayExecutor.create_settings.
min_workers (int): Minimum number of processes running for
training to continue. If number of available processes dips
below this threshold, then training will wait for
more instances to become available.
max_workers (int): Maximum number of training processes,
beyond which no additional processes will be created.
If not specified, then will be unbounded.
reset_limit (int): Maximum number of times that the training
job can scale up or down the number of workers after
which the job is terminated.
cooldown_range (Tuple[int, int]): Range(in seconds) a failing
host will remain in blacklist.
Example: cooldown_range=(10, 100)
This sets the minimum cooldown period to 10 seconds,
and the maximum cooldown period to 100 seconds.
elastic_timeout (int): Timeout for elastic initialisation after
re-scaling the cluster. The default value is 600 seconds.
Alternatively, the environment variable
HOROVOD_ELASTIC_TIMEOUT can also be used.'
cpus_per_worker (int): Number of CPU resources to allocate to
each worker.
use_gpu (bool): Whether to use GPU for allocation. TODO: this
can be removed.
gpus_per_worker (int): Number of GPU resources to allocate to
each worker.
override_discovery (bool): Whether for the ElasticRayExecutor to
automatically provide a discovery mechanism for ElasticSettings.
"""
def __init__(self,
settings,
min_workers: int,
max_workers: Optional[int] = None,
use_gpu: bool = False,
cpus_per_worker: int = 1,
gpus_per_worker: Optional[int] = None,
override_discovery: bool = True,
reset_limit: int = None,
cooldown_range: Optional[Tuple[int, int]] = None,
elastic_timeout: int = 600):
self.settings = settings
if override_discovery:
settings.discovery = RayHostDiscovery(
use_gpu=use_gpu,
cpus_per_worker=cpus_per_worker,
gpus_per_worker=gpus_per_worker)
self.cpus_per_worker = cpus_per_worker
self.gpus_per_worker = gpus_per_worker
self.use_gpu = use_gpu
# moved from settings
self.min_workers = min_workers
self.max_workers = max_workers
self.num_workers = min_workers
self.reset_limit = reset_limit
self.cooldown_range = cooldown_range
self.elastic_timeout = elastic_timeout
self.driver = None
self.rendezvous = None
def start(self,
executable_cls: type = None,
executable_args: Optional[List] = None,
executable_kwargs: Optional[Dict] = None,
extra_env_vars: Optional[Dict] = None):
"""Starts the Horovod driver and services.
Args:
executable_cls (type): The class that will be created within
an actor (BaseHorovodWorker). This will allow Horovod
to establish its connections and set env vars.
executable_args (List): Arguments to be passed into the
worker class upon initialization.
executable_kwargs (Dict): Keyword arguments to be passed into the
worker class upon initialization.
extra_env_vars (Dict): Environment variables to be set
on the actors (worker processes) before initialization.
"""
self.rendezvous = RendezvousServer(self.settings.verbose)
self.driver = ElasticDriver(rendezvous=self.rendezvous,
discovery=self.settings.discovery,
min_np=self.min_workers,
max_np=self.max_workers,
timeout=self.elastic_timeout,
reset_limit=self.reset_limit,
cooldown_range=self.cooldown_range,
verbose=self.settings.verbose)
handler = create_rendezvous_handler(self.driver)
logger.debug("[ray] starting rendezvous")
global_rendezv_port = self.rendezvous.start(handler)
logger.debug(f"[ray] waiting for {self.num_workers} to start.")
self.driver.wait_for_available_slots(self.num_workers)
# Host-to-host common interface detection
# requires at least 2 hosts in an elastic job.
min_hosts = _get_min_start_hosts(self.settings)
current_hosts = self.driver.wait_for_available_slots(
self.num_workers, min_hosts=min_hosts)
logger.debug("[ray] getting common interfaces")
nics = detect_nics(
self.settings,
all_host_names=current_hosts.host_assignment_order,
)
logger.debug("[ray] getting driver IP")
server_ip = socket.gethostbyname(socket.gethostname())
self.run_env_vars = create_run_env_vars(server_ip,
nics,
global_rendezv_port,
elastic=True)
self.executable_cls = executable_cls
self.executable_args = executable_args
self.executable_kwargs = executable_kwargs
self.env_vars = extra_env_vars or {}
def _create_resources(self, hostname: str):
resources = dict(num_cpus=self.cpus_per_worker,
num_gpus=int(self.use_gpu) * self.gpus_per_worker,
resources={f"node:{hostname}": 0.01})
return resources
def _create_remote_worker(self, slot_info, worker_env_vars):
hostname = slot_info.hostname
loaded_worker_cls = self.remote_worker_cls.options(
**self._create_resources(hostname))
worker = loaded_worker_cls.remote()
worker.update_env_vars.remote(worker_env_vars)
worker.update_env_vars.remote(create_slot_env_vars(slot_info))
if self.use_gpu:
visible_devices = ",".join(
[str(i) for i in range(slot_info.local_size)])
worker.update_env_vars.remote(
{"CUDA_VISIBLE_DEVICES": visible_devices})
return worker
def _create_spawn_worker_fn(self, return_results: List,
worker_fn: Callable,
queue: "ray.util.Queue") -> Callable:
self.remote_worker_cls = ray.remote(BaseHorovodWorker)
# event = register_shutdown_event()
worker_env_vars = {}
worker_env_vars.update(self.run_env_vars.copy())
worker_env_vars.update(self.env_vars.copy())
worker_env_vars.update({"PYTHONUNBUFFERED": "1"})
def worker_loop(slot_info, events):
def ping_worker(worker):
# There is an odd edge case where a node can be removed
# before the remote worker is started, leading to a failure
# in trying to create the horovod mesh.
try:
ping = worker.execute.remote(lambda _: 1)
ray.get(ping, timeout=10)
except Exception as e:
logger.error(f"{slot_info.hostname}: Ping failed - {e}")
return False
return True
worker = self._create_remote_worker(slot_info, worker_env_vars)
if not ping_worker(worker):
return 1, time.time()
ray.get(worker.set_queue.remote(queue))
future = worker.execute.remote(worker_fn)
result = None
while result is None:
try:
# TODO: make this event driven at some point.
retval = ray.get(future, timeout=0.1)
return_results.append((slot_info.rank, retval))
# Success
result = 0, time.time()
except GetTimeoutError:
# Timeout
if any(e.is_set() for e in events):
ray.kill(worker)
result = 1, time.time()
except Exception as e:
logger.error(f"{slot_info.hostname}[{slot_info.rank}]:{e}")
ray.kill(worker)
result = 1, time.time()
logger.debug(f"Worker ({slot_info}) routine is done!")
return result
return worker_loop
def run(self,
fn: Callable[[Any], Any],
args: Optional[List] = None,
kwargs: Optional[Dict] = None,
callbacks: Optional[List[Callable]] = None) -> List[Any]:
"""Executes the provided function on all workers.
Args:
fn: Target function that can be executed with arbitrary
args and keyword arguments.
args: List of arguments to be passed into the target function.
kwargs: Dictionary of keyword arguments to be
passed into the target function.
callbacks: List of callables. Each callback must either
be a callable function or a class that implements __call__.
Every callback will be invoked on every value logged
by the rank 0 worker.
Returns:
Deserialized return values from the target function.
"""
args = args or []
kwargs = kwargs or {}
f = lambda _: fn(*args, **kwargs)
return self._run_remote(f, callbacks=callbacks)
def _run_remote(self,
worker_fn: Callable,
callbacks: Optional[List[Callable]] = None) -> List[Any]:
"""Executes the provided function on all workers.
Args:
worker_fn: Target elastic function that can be executed.
callbacks: List of callables. Each callback must either
be a callable function or a class that implements __call__.
Every callback will be invoked on every value logged
by the rank 0 worker.
Returns:
List of return values from every completed worker.
"""
return_values = []
from ray.util.queue import Queue
import inspect
args = inspect.getfullargspec(Queue).args
if "actor_options" not in args:
# Ray 1.1 and less
_queue = Queue()
else:
_queue = Queue(actor_options={
"num_cpus": 0,
"resources": {
ray.state.current_node_id(): 0.001
}
})
self.driver.start(
self.num_workers,
self._create_spawn_worker_fn(return_values, worker_fn, _queue))
def _process_calls(queue, callbacks, event):
if not callbacks:
return
while queue.actor:
if not queue.empty():
result = queue.get_nowait()
for c in callbacks:
c(result)
# avoid slamming the CI
elif event.is_set():
break
time.sleep(0.1)
try:
event = threading.Event()
_callback_thread = threading.Thread(target=_process_calls,
args=(_queue, callbacks,
event),
daemon=True)
_callback_thread.start()
res = self.driver.get_results()
event.set()
if _callback_thread:
_callback_thread.join(timeout=60)
finally:
if hasattr(_queue, "shutdown"):
_queue.shutdown()
else:
done_ref = _queue.actor.__ray_terminate__.remote()
done, not_done = ray.wait([done_ref], timeout=5)
if not_done:
ray.kill(_queue.actor)
self.driver.stop()
if res.error_message is not None:
raise RuntimeError(res.error_message)
for name, value in sorted(res.worker_results.items(),
key=lambda item: item[1][1]):
exit_code, timestamp = value
if exit_code != 0:
raise RuntimeError(
'Horovod detected that one or more processes '
'exited with non-zero '
'status, thus causing the job to be terminated. '
'The first process '
'to do so was:\nProcess name: {name}\nExit code: {code}\n'.
format(name=name, code=exit_code))
return_values = [
value for k, value in sorted(return_values, key=lambda kv: kv[0])
]
return return_values
def run_remote(self, fn: Callable[[Any], Any]) -> List[Any]:
raise NotImplementedError(
"ObjectRefs cannot be returned from Elastic runs as the workers are ephemeral"
)
def execute(self,
fn: Callable[["executable_cls"], Any],
callbacks: Optional[List[Callable]] = None) -> List[Any]:
"""Executes the provided function on all workers.
Args:
fn: Target function to be invoked on every object.
callbacks: List of callables. Each callback must either
be a callable function or a class that implements __call__.
Every callback will be invoked on every value logged
by the rank 0 worker.
Returns:
Deserialized return values from the target function.
"""
return ray.get(self._run_remote(fn, callbacks=callbacks))
def execute_single(self, fn: Callable[["executable_cls"],
Any]) -> List[Any]:
"""Executes the provided function on the rank 0 worker (chief).
Args:
fn: Target function to be invoked on the chief object.
Returns:
Deserialized return values from the target function.
"""
raise NotImplementedError(
"Elastic mode does not support execute_single. Please use the execute method instead"
)
def shutdown(self):
"""Destroys the driver."""
if not self.driver:
return
assert self.driver.finished()
self.driver = None
def test_worker_notification_manager(self):
"""Tests that host add events are sent to the worker notification service and consumed."""
slots = {'host-1': 2}
discovery = FixedHosts(slots)
rendezvous = RendezvousServer()
driver = ElasticDriver(rendezvous, discovery, min_np=2, max_np=4)
driver.wait_for_available_slots(min_np=2)
handler = create_rendezvous_handler(driver)
common_intfs = network.get_local_intfs()
addr = network.get_driver_ip(common_intfs)
port = rendezvous.start(handler)
nic = list(common_intfs)[0]
rank_results = {}
class NotificationReceiver:
def __init__(self):
self.events = []
def on_hosts_updated(self, timestamp, res):
self.events.append((timestamp, res))
def add_host():
slots = {'host-1': 2, 'host-2': 2}
discovery.set(slots)
def remove_host():
slots = {'host-2': 2}
discovery.set(slots)
def exec_command(slot_info, events):
manager = WorkerNotificationManager()
manager.init(rendezvous_addr=addr,
rendezvous_port=port,
nic=nic,
hostname=slot_info.hostname,
local_rank=slot_info.local_rank)
notification_receiver = NotificationReceiver()
manager.register_listener(notification_receiver)
driver.record_ready(slot_info.hostname, slot_info.local_rank)
if slot_info.rank == 0:
add_host()
driver.wait_for_available_slots(4)
if slot_info.rank == 0:
remove_host()
# Busy wait for the number of available slots to decrease
while driver._host_manager.current_hosts.count_available_slots(
) > 2:
time.sleep(0.01)
rank_results[slot_info.rank] = notification_receiver.events
return 0, time.time()
driver.start(np=2, create_worker_fn=exec_command)
res = driver.get_results().worker_results
driver.stop()
assert len(res) == 2
for name, (exit_code, timestamp) in res.items():
assert exit_code == 0, name
assert len(rank_results) == 2
for rank, events in rank_results.items():
expected = 2 if rank == 0 else 0
assert len(events) == expected, rank
if rank == 0:
# First update is an add
assert events[0][1] == HostUpdateResult.added
# Second update is a removal
assert events[1][1] == HostUpdateResult.removed
rendezvous.stop()
