def ray_2_node_4_gpu():
cluster = Cluster()
for _ in range(2):
cluster.add_node(num_cpus=2, num_gpus=4)
ray.init(address=cluster.address)
yield
ray.shutdown()
cluster.shutdown()
def ray_4_node_8_cpu():
cluster = Cluster()
for _ in range(4):
cluster.add_node(num_cpus=8)
ray.init(address=cluster.address)
yield
ray.shutdown()
cluster.shutdown()
def test_ray_init_using_hostname(ray_start_cluster):
import socket
hostname = socket.gethostname()
cluster = Cluster(
initialize_head=True,
head_node_args={
"node_ip_address": hostname,
},
)
# Use `ray.init` to test the connection.
ray.init(address=cluster.address, _node_ip_address=hostname)
node_table = cluster.global_state.node_table()
assert len(node_table) == 1
assert node_table[0].get("NodeManagerHostname", "") == hostname
def ray_start_workers_separate_multinode(request):
num_nodes = request.param[0]
num_initial_workers = request.param[1]
# Start the Ray processes.
cluster = Cluster()
for _ in range(num_nodes):
cluster.add_node(num_cpus=num_initial_workers)
ray.init(address=cluster.address)
yield num_nodes, num_initial_workers
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
async def test_asyncio_cluster_wait():
cluster = Cluster()
head_node = cluster.add_node()
cluster.add_node(resources={"OTHER_NODE": 100})
ray.init(address=head_node.address)
@ray.remote(num_cpus=0, resources={"OTHER_NODE": 1})
def get_array():
return np.random.random((192, 1080, 3)).astype(np.uint8) # ~ 0.5MB
object_ref = get_array.remote()
await asyncio.wait_for(object_ref, timeout=10)
ray.shutdown()
cluster.shutdown()
def ray_4_node_gpu():
cluster = Cluster()
for _ in range(4):
cluster.add_node(num_cpus=2, num_gpus=2)
ray.init(address=cluster.address)
yield
ray.shutdown()
cluster.shutdown()
# Ensure that tests don't ALL fail
if dist.is_initialized():
dist.destroy_process_group()
def test_task_args_memory_threshold(shutdown_only, reservation):
num_objs = 3
obj_size = int(1e8 // num_objs)
cluster = Cluster()
cluster.add_node(num_cpus=0,
_system_config={
"pull_manager_memory_fraction": reservation / 10,
},
object_store_memory=1e9)
ray.init(address=cluster.address)
cluster.add_node(num_cpus=num_objs, object_store_memory=1e9)
@ray.remote
class Signal:
def __init__(self, num_events):
self.num_events = num_events
self.ready_event = asyncio.Event()
def send(self):
self.num_events -= 1
if self.num_events == 0:
self.ready_event.set()
async def wait(self):
await self.ready_event.wait()
@ray.remote
def f(actor, arg):
actor.send.remote()
time.sleep(1000)
@ray.remote
def empty():
return
num_tasks_expected = int(reservation / (1 / num_objs))
print(num_tasks_expected)
a = Signal.remote(num_tasks_expected)
x = np.zeros(obj_size, dtype=np.uint8)
objs = [ray.put(x) for _ in range(num_objs)]
# The tasks have to fetch the objects to the remote node to run. The remote
# node should only fetch objects up to the configured reservation.
for obj in objs:
f.remote(a, obj)
# Check that at least num_tasks_expected tasks are scheduled.
ray.get(a.wait.remote(), timeout=10)
# Check that at most num_tasks_expected tasks are scheduled.
num_cores = num_objs - num_tasks_expected
ray.get(empty.options(num_cpus=num_cores).remote(), timeout=10)
def test_namespace():
"""
Most of the "checks" in this test case rely on the fact that
`run_string_as_driver` will throw an exception if the driver string exits
with a non-zero exit code (e.g. when the driver scripts throws an
exception). Since all of these drivers start named, detached actors, the
most likely failure case would be a collision of named actors if they're
put in the same namespace.
This test checks that:
* When two drivers don't specify a namespace, they are placed in different
anonymous namespaces.
* When two drivers specify a namespace, they collide.
* The namespace name (as provided by the runtime context) is correct.
"""
cluster = Cluster()
cluster.add_node(num_cpus=4, ray_client_server_port=50055)
cluster.wait_for_nodes(1)
template = """
import ray
ray.client("localhost:50055").namespace({namespace}).connect()
@ray.remote
class Foo:
def ping(self):
return "pong"
a = Foo.options(lifetime="detached", name="abc").remote()
ray.get(a.ping.remote())
print(ray.get_runtime_context().namespace)
"""
anon_driver = template.format(namespace="None")
run_string_as_driver(anon_driver)
# This second run will fail if the actors don't run in separate anonymous
# namespaces.
run_string_as_driver(anon_driver)
run_in_namespace = template.format(namespace="'namespace'")
script_namespace = run_string_as_driver(run_in_namespace)
# The second run fails because the actors are run in the same namespace.
with pytest.raises(subprocess.CalledProcessError):
run_string_as_driver(run_in_namespace)
assert script_namespace.strip() == "namespace"
subprocess.check_output("ray stop --force", shell=True)
def test_ray_status_multinode():
cluster = Cluster()
for _ in range(4):
cluster.add_node(num_cpus=2)
runner = CliRunner()
def output_ready():
result = runner.invoke(scripts.status)
result.stdout
return not result.exception and "memory" in result.output
wait_for_condition(output_ready)
result = runner.invoke(scripts.status, [])
_check_output_via_pattern("test_ray_status_multinode.txt", result)
ray.shutdown()
cluster.shutdown()
def ray_start_combination(request):
num_nodes = request.param[0]
num_workers_per_scheduler = request.param[1]
# Start the Ray processes.
cluster = Cluster(initialize_head=True,
head_node_args={
"num_cpus": 10,
"redis_max_memory": 10**7
})
for i in range(num_nodes - 1):
cluster.add_node(num_cpus=10)
ray.init(address=cluster.address)
yield num_nodes, num_workers_per_scheduler, cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def test_raylet_socket_name(shutdown_only):
sock1 = unix_socket_create_path("i_am_a_temp_socket_1")
ray.init(raylet_socket_name=sock1)
unix_socket_verify(sock1)
ray.shutdown()
try:
unix_socket_delete(sock1)
except OSError:
pass # It could have been removed by Ray.
cluster = Cluster(True)
sock2 = unix_socket_create_path("i_am_a_temp_socket_2")
cluster.add_node(raylet_socket_name=sock2)
unix_socket_verify(sock2)
cluster.shutdown()
try:
unix_socket_delete(sock2)
except OSError:
pass # It could have been removed by Ray.
def test_temp_plasma_store_socket():
ray.init(plasma_store_socket_name="/tmp/i_am_a_temp_socket")
assert os.path.exists(
"/tmp/i_am_a_temp_socket"), "Specified socket path not found."
ray.shutdown()
try:
os.remove("/tmp/i_am_a_temp_socket")
except OSError:
pass # It could have been removed by Ray.
cluster = Cluster(True)
cluster.add_node(plasma_store_socket_name="/tmp/i_am_a_temp_socket_2")
assert os.path.exists(
"/tmp/i_am_a_temp_socket_2"), "Specified socket path not found."
cluster.shutdown()
try:
os.remove("/tmp/i_am_a_temp_socket_2")
except OSError:
pass # It could have been removed by Ray.
def test_system_config_when_connecting(ray_start_cluster):
config = {"object_timeout_milliseconds": 200}
cluster = Cluster()
cluster.add_node(_system_config=config, object_store_memory=100 * 1024 * 1024)
cluster.wait_for_nodes()
# Specifying _system_config when connecting to a cluster is disallowed.
with pytest.raises(ValueError):
ray.init(address=cluster.address, _system_config=config)
# Check that the config was picked up (object pinning is disabled).
ray.init(address=cluster.address)
obj_ref = ray.put(np.zeros(40 * 1024 * 1024, dtype=np.uint8))
for _ in range(5):
put_ref = ray.put(np.zeros(40 * 1024 * 1024, dtype=np.uint8))
del put_ref
ray.get(obj_ref)
def test_cluster_handle_affinity():
cluster = Cluster()
# HACK: using two different ip address so the placement constraint for
# resource check later will work.
head_node = cluster.add_node(node_ip_address="127.0.0.1", num_cpus=4)
cluster.add_node(node_ip_address="0.0.0.0", num_cpus=4)
ray.init(head_node.address)
# Make sure we have two nodes.
node_ids = [n["NodeID"] for n in ray.nodes()]
assert len(node_ids) == 2
# Start the backend.
client = serve.start(http_port=randint(10000, 30000), detached=True)
client.create_backend("hi:v0", lambda _: "hi")
client.create_endpoint("hi", backend="hi:v0")
# Try to retrieve the handle from both head and worker node, check the
# router's node id.
@ray.remote
def check_handle_router_id():
client = serve.connect()
handle = client.get_handle("hi")
return get_node_id_for_actor(handle.router_handle)
router_node_ids = ray.get([
check_handle_router_id.options(resources={
node_id: 0.01
}).remote() for node_id in ray.state.node_ids()
])
assert set(router_node_ids) == set(node_ids)
# Clean up the nodes (otherwise Ray will segfault).
ray.shutdown()
cluster.shutdown()
def main():
cluster = Cluster(
initialize_head=True,
connect=True,
head_node_args={"object_store_memory": 20 * 1024 * 1024 * 1024, "num_cpus": 16},
)
cluster.add_node(
object_store_memory=20 * 1024 * 1024 * 1024, num_gpus=1, num_cpus=16
)
object_ref_list = []
for i in range(0, 10):
object_ref = ray.put(np.random.rand(1024 * 128, 1024))
object_ref_list.append(object_ref)
@ray.remote(num_gpus=1)
def f(object_ref_list):
diffs = []
for object_ref in object_ref_list:
before = time.time()
ray.get(object_ref)
after = time.time()
diffs.append(after - before)
time.sleep(1)
return np.mean(diffs), np.std(diffs)
time_diff, time_diff_std = ray.get(f.remote(object_ref_list))
print(
"latency to get an 1G object over network",
round(time_diff, 2),
"+-",
round(time_diff_std, 2),
)
ray.shutdown()
cluster.shutdown()
def test_fate_sharing(ray_start_cluster, use_actors, node_failure):
config = {
"num_heartbeats_timeout": 10,
"raylet_heartbeat_period_milliseconds": 100,
}
cluster = Cluster()
# Head node with no resources.
cluster.add_node(num_cpus=0, _system_config=config)
ray.init(address=cluster.address)
# Node to place the parent actor.
node_to_kill = cluster.add_node(num_cpus=1, resources={"parent": 1})
# Node to place the child actor.
cluster.add_node(num_cpus=1, resources={"child": 1})
cluster.wait_for_nodes()
@ray.remote
def sleep():
time.sleep(1000)
@ray.remote(resources={"child": 1})
def probe():
return
# TODO(swang): This test does not pass if max_restarts > 0 for the
# raylet codepath. Add this parameter once the GCS actor service is enabled
# by default.
@ray.remote
class Actor(object):
def __init__(self):
return
def start_child(self, use_actors):
if use_actors:
child = Actor.options(resources={"child": 1}).remote()
ray.get(child.sleep.remote())
else:
ray.get(sleep.options(resources={"child": 1}).remote())
def sleep(self):
time.sleep(1000)
def get_pid(self):
return os.getpid()
# Returns whether the "child" resource is available.
def child_resource_available():
p = probe.remote()
ready, _ = ray.wait([p], timeout=1)
return len(ready) > 0
# Test fate sharing if the parent process dies.
def test_process_failure(use_actors):
a = Actor.options(resources={"parent": 1}).remote()
pid = ray.get(a.get_pid.remote())
a.start_child.remote(use_actors=use_actors)
# Wait for the child to be scheduled.
wait_for_condition(lambda: not child_resource_available())
# Kill the parent process.
os.kill(pid, 9)
wait_for_condition(child_resource_available)
# Test fate sharing if the parent node dies.
def test_node_failure(node_to_kill, use_actors):
a = Actor.options(resources={"parent": 1}).remote()
a.start_child.remote(use_actors=use_actors)
# Wait for the child to be scheduled.
wait_for_condition(lambda: not child_resource_available())
# Kill the parent process.
cluster.remove_node(node_to_kill, allow_graceful=False)
node_to_kill = cluster.add_node(num_cpus=1, resources={"parent": 1})
wait_for_condition(child_resource_available)
return node_to_kill
if node_failure:
test_node_failure(node_to_kill, use_actors)
else:
test_process_failure(use_actors)
class TrialRunnerPlacementGroupTest(unittest.TestCase):
def setUp(self):
os.environ["TUNE_GLOBAL_CHECKPOINT_S"] = "10000"
self.head_cpus = 8
self.head_gpus = 4
self.head_custom = 16
self.cluster = Cluster(initialize_head=True,
connect=True,
head_node_args={
"include_dashboard": False,
"num_cpus": self.head_cpus,
"num_gpus": self.head_gpus,
"resources": {
"custom": self.head_custom
},
"_system_config": {
"num_heartbeats_timeout": 10
}
})
# Pytest doesn't play nicely with imports
_register_all()
def tearDown(self):
ray.shutdown()
self.cluster.shutdown()
_register_all() # re-register the evicted objects
def _assertCleanup(self, trial_executor):
# Assert proper cleanup
pg_manager = trial_executor._pg_manager
self.assertFalse(pg_manager._in_use_trials)
self.assertFalse(pg_manager._in_use_pgs)
self.assertFalse(pg_manager._staging_futures)
for pgf in pg_manager._staging:
self.assertFalse(pg_manager._staging[pgf])
for pgf in pg_manager._ready:
self.assertFalse(pg_manager._ready[pgf])
self.assertTrue(pg_manager._latest_staging_start_time)
num_non_removed_pgs = len([
p for pid, p in placement_group_table().items()
if p["state"] != "REMOVED"
])
self.assertEqual(num_non_removed_pgs, 0)
def testPlacementGroupRequests(self, reuse_actors=False, scheduled=10):
"""In this test we try to start 10 trials but only have resources
for 2. Placement groups should still be created and PENDING.
Eventually they should be scheduled sequentially (i.e. in pairs
of two)."""
def train(config):
time.sleep(1)
now = time.time()
tune.report(end=now - config["start_time"])
head_bundle = {"CPU": 4, "GPU": 0, "custom": 0}
child_bundle = {"custom": 1}
placement_group_factory = PlacementGroupFactory(
[head_bundle, child_bundle, child_bundle])
trial_executor = RayTrialExecutor(reuse_actors=reuse_actors)
this = self
class _TestCallback(Callback):
def on_step_end(self, iteration, trials, **info):
num_finished = len([
t for t in trials
if t.status == Trial.TERMINATED or t.status == Trial.ERROR
])
num_staging = sum(
len(s)
for s in trial_executor._pg_manager._staging.values())
num_ready = sum(
len(s) for s in trial_executor._pg_manager._ready.values())
num_in_use = len(trial_executor._pg_manager._in_use_pgs)
num_cached = len(trial_executor._pg_manager._cached_pgs)
total_num_tracked = num_staging + num_ready + \
num_in_use + num_cached
num_non_removed_pgs = len([
p for pid, p in placement_group_table().items()
if p["state"] != "REMOVED"
])
num_removal_scheduled_pgs = len(
trial_executor._pg_manager._pgs_for_removal)
# All trials should be scheduled
this.assertEqual(scheduled,
min(scheduled, len(trials)),
msg=f"Num trials iter {iteration}")
# The number of PGs should decrease when trials finish
this.assertEqual(max(scheduled, len(trials)) - num_finished,
total_num_tracked,
msg=f"Num tracked iter {iteration}")
# The number of actual placement groups should match this
this.assertEqual(max(scheduled, len(trials)) - num_finished,
num_non_removed_pgs -
num_removal_scheduled_pgs,
msg=f"Num actual iter {iteration}")
start = time.time()
out = tune.run(train,
config={"start_time": start},
resources_per_trial=placement_group_factory,
num_samples=10,
trial_executor=trial_executor,
callbacks=[_TestCallback()],
reuse_actors=reuse_actors,
verbose=2)
trial_end_times = sorted(t.last_result["end"] for t in out.trials)
print("Trial end times:", trial_end_times)
max_diff = trial_end_times[-1] - trial_end_times[0]
# Not all trials have been run in parallel
self.assertGreater(max_diff, 3)
# Some trials should have run in parallel
# Todo: Re-enable when using buildkite
# self.assertLess(max_diff, 10)
self._assertCleanup(trial_executor)
def testPlacementGroupRequestsWithActorReuse(self):
"""Assert that reuse actors doesn't leak placement groups"""
self.testPlacementGroupRequests(reuse_actors=True)
@patch("ray.tune.trial_runner.TUNE_MAX_PENDING_TRIALS_PG", 6)
@patch("ray.tune.utils.placement_groups.TUNE_MAX_PENDING_TRIALS_PG", 6)
def testPlacementGroupLimitedRequests(self):
"""Assert that maximum number of placement groups is enforced."""
self.testPlacementGroupRequests(scheduled=6)
@patch("ray.tune.trial_runner.TUNE_MAX_PENDING_TRIALS_PG", 6)
@patch("ray.tune.utils.placement_groups.TUNE_MAX_PENDING_TRIALS_PG", 6)
def testPlacementGroupLimitedRequestsWithActorReuse(self):
self.testPlacementGroupRequests(reuse_actors=True, scheduled=6)
def testPlacementGroupDistributedTraining(self, reuse_actors=False):
"""Run distributed training using placement groups.
Each trial requests 4 CPUs and starts 4 remote training workers.
"""
head_bundle = {"CPU": 1, "GPU": 0, "custom": 0}
child_bundle = {"CPU": 1}
placement_group_factory = PlacementGroupFactory(
[head_bundle, child_bundle, child_bundle, child_bundle])
@ray.remote
class TrainingActor:
def train(self, val):
time.sleep(1)
return val
def train(config):
base = config["base"]
actors = [TrainingActor.remote() for _ in range(4)]
futures = [
actor.train.remote(base + 2 * i)
for i, actor in enumerate(actors)
]
results = ray.get(futures)
end = time.time() - config["start_time"]
tune.report(avg=np.mean(results), end=end)
trial_executor = RayTrialExecutor(reuse_actors=reuse_actors)
start = time.time()
out = tune.run(train,
config={
"start_time": start,
"base": tune.grid_search(list(range(0, 100, 10)))
},
resources_per_trial=placement_group_factory,
num_samples=1,
trial_executor=trial_executor,
reuse_actors=reuse_actors,
verbose=2)
avgs = sorted(t.last_result["avg"] for t in out.trials)
self.assertSequenceEqual(avgs, list(range(3, 103, 10)))
trial_end_times = sorted(t.last_result["end"] for t in out.trials)
print("Trial end times:", trial_end_times)
max_diff = trial_end_times[-1] - trial_end_times[0]
# Not all trials have been run in parallel
self.assertGreater(max_diff, 3)
# Some trials should have run in parallel
# Todo: Re-enable when using buildkite
# self.assertLess(max_diff, 10)
self._assertCleanup(trial_executor)
def testPlacementGroupDistributedTrainingWithActorReuse(self):
self.testPlacementGroupDistributedTraining(reuse_actors=True)
def test_connect_with_disconnected_node(shutdown_only):
config = json.dumps({
"num_heartbeats_timeout": 50,
"raylet_heartbeat_timeout_milliseconds": 10,
})
cluster = Cluster()
cluster.add_node(num_cpus=0, _internal_config=config)
ray.init(address=cluster.address)
info = relevant_errors(ray_constants.REMOVED_NODE_ERROR)
assert len(info) == 0
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(dead_node, allow_graceful=False)
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 1)
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(dead_node, allow_graceful=False)
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2)
# This node is killed by SIGTERM, ray_monitor will not mark it again.
removing_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(removing_node, allow_graceful=True)
with pytest.raises(RayTestTimeoutException):
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 3, timeout=2)
# There is no connection error to a dead node.
info = relevant_errors(ray_constants.RAYLET_CONNECTION_ERROR)
assert len(info) == 0
def ray_cluster():
cluster = Cluster()
yield Cluster()
serve.shutdown()
ray.shutdown()
cluster.shutdown()
class RayExecutorQueueTest(unittest.TestCase):
def setUp(self):
self.cluster = Cluster(initialize_head=True,
connect=True,
head_node_args={
"num_cpus": 1,
"_system_config": {
"num_heartbeats_timeout": 10
}
})
self.trial_executor = RayTrialExecutor(queue_trials=True,
refresh_period=0)
# Pytest doesn't play nicely with imports
_register_all()
def tearDown(self):
ray.shutdown()
self.cluster.shutdown()
_register_all() # re-register the evicted objects
def testQueueTrial(self):
"""Tests that reset handles NotImplemented properly."""
def create_trial(cpu, gpu=0):
return Trial("__fake", resources=Resources(cpu=cpu, gpu=gpu))
cpu_only = create_trial(1, 0)
self.assertTrue(self.trial_executor.has_resources_for_trial(cpu_only))
self.trial_executor.start_trial(cpu_only)
gpu_only = create_trial(0, 1)
self.assertTrue(self.trial_executor.has_resources_for_trial(gpu_only))
def testHeadBlocking(self):
# Once resource requests are deprecated, remove this test
os.environ["TUNE_PLACEMENT_GROUP_AUTO_DISABLED"] = "1"
def create_trial(cpu, gpu=0):
return Trial("__fake", resources=Resources(cpu=cpu, gpu=gpu))
gpu_trial = create_trial(1, 1)
self.assertTrue(self.trial_executor.has_resources_for_trial(gpu_trial))
self.trial_executor.start_trial(gpu_trial)
# TODO(rliaw): This behavior is probably undesirable, but right now
# trials with different resource requirements is not often used.
cpu_only_trial = create_trial(1, 0)
self.assertFalse(
self.trial_executor.has_resources_for_trial(cpu_only_trial))
self.cluster.add_node(num_cpus=1, num_gpus=1)
self.cluster.wait_for_nodes()
self.assertTrue(
self.trial_executor.has_resources_for_trial(cpu_only_trial))
self.trial_executor.start_trial(cpu_only_trial)
cpu_only_trial2 = create_trial(1, 0)
self.assertTrue(
self.trial_executor.has_resources_for_trial(cpu_only_trial2))
self.trial_executor.start_trial(cpu_only_trial2)
cpu_only_trial3 = create_trial(1, 0)
self.assertFalse(
self.trial_executor.has_resources_for_trial(cpu_only_trial3))
def test_spill_dir_cleanup_on_raylet_start(object_spilling_config):
object_spilling_config, temp_folder = object_spilling_config
cluster = Cluster()
cluster.add_node(
num_cpus=0,
object_store_memory=75 * 1024 * 1024,
_system_config={"object_spilling_config": object_spilling_config},
)
ray.init(address=cluster.address)
node2 = cluster.add_node(num_cpus=1, object_store_memory=75 * 1024 * 1024)
# This task will run on node 2 because node 1 has no CPU resource
@ray.remote(num_cpus=1)
def run_workload():
ids = []
for _ in range(2):
arr = np.random.rand(5 * 1024 * 1024) # 40 MB
ids.append(ray.put(arr))
return ids
ids = ray.get(run_workload.remote())
assert not is_dir_empty(temp_folder)
# Kill node 2
cluster.remove_node(node2)
# Verify that the spill folder is not empty
assert not is_dir_empty(temp_folder)
# Start a new node
cluster.add_node(num_cpus=1, object_store_memory=75 * 1024 * 1024)
# Verify that the spill folder is now cleaned up
assert is_dir_empty(temp_folder)
# We hold the object refs to prevent them from being deleted
del ids
ray.shutdown()
cluster.shutdown()
def test_shutdown():
g = Cluster(initialize_head=False)
node = g.add_node()
node2 = g.add_node()
g.shutdown()
assert not any(n.any_processes_alive() for n in [node, node2])
json.dump(result, f)
num_redis_shards = 5
redis_max_memory = 10**8
object_store_memory = 10**8
num_nodes = 10
message = ("Make sure there is enough memory on this machine to run this "
"workload. We divide the system memory by 2 to provide a buffer.")
assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory <
ray._private.utils.get_system_memory() / 2), message
# Simulate a cluster on one machine.
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(
redis_port=6379 if i == 0 else None,
num_redis_shards=num_redis_shards if i == 0 else None,
num_cpus=2,
num_gpus=0,
resources={str(i): 2},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory,
dashboard_host="0.0.0.0",
)
ray.init(address=cluster.address)
# Run the workload.
class RayExecutorPlacementGroupTest(unittest.TestCase):
def setUp(self):
self.head_cpus = 8
self.head_gpus = 4
self.head_custom = 16
self.cluster = Cluster(
initialize_head=True,
connect=True,
head_node_args={
"num_cpus": self.head_cpus,
"num_gpus": self.head_gpus,
"resources": {
"custom": self.head_custom
},
"_system_config": {
"num_heartbeats_timeout": 10
},
},
)
# Pytest doesn't play nicely with imports
_register_all()
def tearDown(self):
ray.shutdown()
self.cluster.shutdown()
_register_all() # re-register the evicted objects
def testResourcesAvailableWithPlacementGroup(self):
def train(config):
tune.report(metric=0, resources=ray.available_resources())
head_bundle = {"CPU": 1, "GPU": 0, "custom": 4}
child_bundle = {"CPU": 2, "GPU": 1, "custom": 3}
placement_group_factory = PlacementGroupFactory(
[head_bundle, child_bundle, child_bundle])
out = tune.run(train, resources_per_trial=placement_group_factory)
available = {
key: val
for key, val in out.trials[0].last_result["resources"].items()
if key in ["CPU", "GPU", "custom"]
}
if not available:
self.skipTest("Warning: Ray reported no available resources, "
"but this is an error on the Ray core side. "
"Skipping this test for now.")
self.assertDictEqual(
available,
{
"CPU": self.head_cpus - 5.0,
"GPU": self.head_gpus - 2.0,
"custom": self.head_custom - 10.0,
},
)
def testPlacementGroupFactoryEquality(self):
"""
Test that two different placement group factory objects are considered
equal and evaluate to the same hash.
"""
from collections import Counter
pgf_1 = PlacementGroupFactory(
[{
"CPU": 2,
"GPU": 4,
"custom": 7
}, {
"GPU": 2,
"custom": 1,
"CPU": 3
}],
"PACK",
"no_name",
None,
)
pgf_2 = PlacementGroupFactory(
[
{
"custom": 7,
"GPU": 4,
"CPU": 2,
},
{
"custom": 1,
"GPU": 2,
"CPU": 3
},
],
strategy="PACK",
name="no_name",
lifetime=None,
)
pgf_3 = PlacementGroupFactory(
[
{
"custom": 7,
"GPU": 4,
"CPU": 2.0,
"custom2": 0
},
{
"custom": 1.0,
"GPU": 2,
"CPU": 3,
"custom2": 0
},
],
strategy="PACK",
name="no_name",
lifetime=None,
)
self.assertEqual(pgf_1, pgf_2)
self.assertEqual(pgf_2, pgf_3)
# Hash testing
counter = Counter()
counter[pgf_1] += 1
counter[pgf_2] += 1
counter[pgf_3] += 1
self.assertEqual(counter[pgf_1], 3)
self.assertEqual(counter[pgf_2], 3)
self.assertEqual(counter[pgf_3], 3)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"checkpoint_at_end": args.checkpoint_at_end,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial
and resources_to_json(args.resources_per_trial)
),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"sync_config": {
"upload_dir": args.upload_dir,
},
}
}
# Ray UI.
if args.no_ray_ui:
deprecation_warning(old="--no-ray-ui", new="--ray-ui", error=False)
args.ray_ui = False
verbose = 1
for exp in experiments.values():
# Bazel makes it hard to find files specified in `args` (and `data`).
# Look for them here.
# NOTE: Some of our yaml files don't have a `config` section.
input_ = exp.get("config", {}).get("input")
if input_ and input_ != "sampler":
# This script runs in the ray/rllib dir.
rllib_dir = Path(__file__).parent
def patch_path(path):
if isinstance(path, list):
return [patch_path(i) for i in path]
elif isinstance(path, dict):
return {
patch_path(k): patch_path(v)
for k, v in path.items()
}
elif isinstance(path, str):
if os.path.exists(path):
return path
else:
abs_path = str(rllib_dir.absolute().joinpath(path))
return abs_path if os.path.exists(abs_path) else path
else:
return path
exp["config"]["input"] = patch_path(input_)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.torch:
deprecation_warning("--torch", "--framework=torch")
exp["config"]["framework"] = "torch"
elif args.eager:
deprecation_warning("--eager", "--framework=[tf2|tfe]")
exp["config"]["framework"] = "tfe"
elif args.framework is not None:
exp["config"]["framework"] = args.framework
if args.trace:
if exp["config"]["framework"] not in ["tf2", "tfe"]:
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.v:
exp["config"]["log_level"] = "INFO"
verbose = 3 # Print details on trial result
if args.vv:
exp["config"]["log_level"] = "DEBUG"
verbose = 3 # Print details on trial result
if args.ray_num_nodes:
# Import this only here so that train.py also works with
# older versions (and user doesn't use `--ray-num-nodes`).
from ray.cluster_utils import Cluster
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
)
ray.init(address=cluster.address)
else:
ray.init(
include_dashboard=args.ray_ui,
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
local_mode=args.local_mode,
)
if IS_NOTEBOOK:
progress_reporter = JupyterNotebookReporter(
overwrite=verbose >= 3, print_intermediate_tables=verbose >= 1)
else:
progress_reporter = CLIReporter(print_intermediate_tables=verbose >= 1)
run_experiments(
experiments,
scheduler=create_scheduler(args.scheduler, **args.scheduler_config),
resume=args.resume,
verbose=verbose,
progress_reporter=progress_reporter,
concurrent=True,
)
ray.shutdown()
class TrialRunnerPlacementGroupTest(unittest.TestCase):
def setUp(self):
os.environ["TUNE_GLOBAL_CHECKPOINT_S"] = "10000"
self.head_cpus = 8
self.head_gpus = 4
self.head_custom = 16
self.cluster = Cluster(initialize_head=True,
connect=True,
head_node_args={
"num_cpus": self.head_cpus,
"num_gpus": self.head_gpus,
"resources": {
"custom": self.head_custom
},
"_system_config": {
"num_heartbeats_timeout": 10
}
})
# Pytest doesn't play nicely with imports
_register_all()
def tearDown(self):
ray.shutdown()
self.cluster.shutdown()
_register_all() # re-register the evicted objects
def testPlacementGroupRequests(self, scheduled=10):
"""In this test we try to start 10 trials but only have resources
for 2. Placement groups should still be created and PENDING.
Eventually they should be scheduled sequentially (i.e. in pairs
of two)."""
def train(config):
time.sleep(1)
now = time.time()
tune.report(end=now - config["start_time"])
def placement_group_factory():
head_bundle = {"CPU": 4, "GPU": 0, "custom": 0}
child_bundle = {"custom": 1}
return placement_group([head_bundle, child_bundle, child_bundle])
trial_executor = RayTrialExecutor()
this = self
class _TestCallback(Callback):
def on_step_end(self, iteration, trials, **info):
if iteration == 1:
this.assertEqual(scheduled, len(trials))
this.assertEqual(
scheduled,
sum(
len(s) for s in
trial_executor._pg_manager._staging.values()) +
sum(
len(s) for s in
trial_executor._pg_manager._ready.values()) +
len(trial_executor._pg_manager._in_use_pgs))
start = time.time()
out = tune.run(train,
config={"start_time": start},
resources_per_trial=placement_group_factory,
num_samples=10,
trial_executor=trial_executor,
callbacks=[_TestCallback()])
trial_end_times = sorted(t.last_result["end"] for t in out.trials)
print("Trial end times:", trial_end_times)
max_diff = trial_end_times[-1] - trial_end_times[0]
# Not all trials have been run in parallel
self.assertGreater(max_diff, 5)
# Some trials should have run in parallel
self.assertLess(max_diff, 10)
@patch("ray.tune.trial_runner.TUNE_MAX_PENDING_TRIALS_PG", 6)
@patch("ray.tune.utils.placement_groups.TUNE_MAX_PENDING_TRIALS_PG", 6)
def testPlacementGroupLimitedRequests(self):
"""Assert that maximum number of placement groups is enforced."""
self.testPlacementGroupRequests(scheduled=6)
def testPlacementGroupDistributedTraining(self):
"""Run distributed training using placement groups.
Each trial requests 4 CPUs and starts 4 remote training workers.
"""
def placement_group_factory():
head_bundle = {"CPU": 1, "GPU": 0, "custom": 0}
child_bundle = {"CPU": 1}
return placement_group(
[head_bundle, child_bundle, child_bundle, child_bundle])
@ray.remote
class TrainingActor:
def train(self, val):
time.sleep(1)
return val
def train(config):
base = config["base"]
actors = [TrainingActor.remote() for _ in range(4)]
futures = [
actor.train.remote(base + 2 * i)
for i, actor in enumerate(actors)
]
results = ray.get(futures)
end = time.time() - config["start_time"]
tune.report(avg=np.mean(results), end=end)
trial_executor = RayTrialExecutor()
start = time.time()
out = tune.run(train,
config={
"start_time": start,
"base": tune.grid_search(list(range(0, 100, 10)))
},
resources_per_trial=placement_group_factory,
num_samples=1,
trial_executor=trial_executor)
avgs = sorted(t.last_result["avg"] for t in out.trials)
self.assertSequenceEqual(avgs, list(range(3, 103, 10)))
trial_end_times = sorted(t.last_result["end"] for t in out.trials)
print("Trial end times:", trial_end_times)
max_diff = trial_end_times[-1] - trial_end_times[0]
# Not all trials have been run in parallel
self.assertGreater(max_diff, 5)
# Some trials should have run in parallel
# Todo: Re-enable when using buildkite
# self.assertLess(max_diff, 10)
# Assert proper cleanup
pg_manager = trial_executor._pg_manager
self.assertFalse(pg_manager._in_use_trials)
self.assertFalse(pg_manager._in_use_pgs)
self.assertFalse(pg_manager._staging_futures)
for pgf in pg_manager._staging:
self.assertFalse(pg_manager._staging[pgf])
for pgf in pg_manager._ready:
self.assertFalse(pg_manager._ready[pgf])
self.assertTrue(pg_manager._latest_staging_start_time)
import ray
from ray.cluster_utils import Cluster
num_redis_shards = 5
redis_max_memory = 10**8
object_store_memory = 10**8
num_nodes = 10
message = ("Make sure there is enough memory on this machine to run this "
"workload. We divide the system memory by 2 to provide a buffer.")
assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory <
ray.utils.get_system_memory() / 2), message
# Simulate a cluster on one machine.
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(redis_port=6379 if i == 0 else None,
num_redis_shards=num_redis_shards if i == 0 else None,
num_cpus=2,
num_gpus=0,
resources={str(i): 2},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory,
webui_host="0.0.0.0")
ray.init(address=cluster.address)
# Run the workload.
@ray.remote
def test_multiple_routers():
cluster = Cluster()
head_node = cluster.add_node()
cluster.add_node()
ray.init(head_node.address)
node_ids = ray.state.node_ids()
assert len(node_ids) == 2
client = serve.start(http_port=8005) # noqa: F841
def get_proxy_names():
proxy_names = []
for node_id, _ in get_all_node_ids():
proxy_names.append(
format_actor_name(SERVE_PROXY_NAME, client._controller_name,
node_id))
return proxy_names
wait_for_condition(lambda: len(get_proxy_names()) == 2)
proxy_names = get_proxy_names()
# Two actors should be started.
def get_first_two_actors():
try:
ray.get_actor(proxy_names[0])
ray.get_actor(proxy_names[1])
return True
except ValueError:
return False
wait_for_condition(get_first_two_actors)
# Wait for the actors to come up.
ray.get(block_until_http_ready.remote("http://127.0.0.1:8005/-/routes"))
# Kill one of the servers, the HTTP server should still function.
ray.kill(ray.get_actor(get_proxy_names()[0]), no_restart=True)
ray.get(block_until_http_ready.remote("http://127.0.0.1:8005/-/routes"))
# Add a new node to the cluster. This should trigger a new router to get
# started.
new_node = cluster.add_node()
wait_for_condition(lambda: len(get_proxy_names()) == 3)
third_proxy = get_proxy_names()[2]
def get_third_actor():
try:
ray.get_actor(third_proxy)
return True
# IndexErrors covers when cluster resources aren't updated yet.
except (IndexError, ValueError):
return False
wait_for_condition(get_third_actor)
# Remove the newly-added node from the cluster. The corresponding actor
# should be removed as well.
cluster.remove_node(new_node)
def third_actor_removed():
try:
ray.get_actor(third_proxy)
return False
except ValueError:
return True
# Check that the actor is gone and the HTTP server still functions.
wait_for_condition(third_actor_removed)
ray.get(block_until_http_ready.remote("http://127.0.0.1:8005/-/routes"))
# Clean up the nodes (otherwise Ray will segfault).
ray.shutdown()
cluster.shutdown()
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
verbose = 1
for exp in experiments.values():
# Bazel makes it hard to find files specified in `args` (and `data`).
# Look for them here.
# NOTE: Some of our yaml files don't have a `config` section.
if exp.get("config", {}).get("input") and \
not os.path.exists(exp["config"]["input"]):
# This script runs in the ray/rllib dir.
rllib_dir = Path(__file__).parent
input_file = rllib_dir.absolute().joinpath(exp["config"]["input"])
exp["config"]["input"] = str(input_file)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.eager:
exp["config"]["eager"] = True
if args.torch:
exp["config"]["use_pytorch"] = True
if args.v:
exp["config"]["log_level"] = "INFO"
verbose = 2
if args.vv:
exp["config"]["log_level"] = "DEBUG"
verbose = 3
if args.trace:
if not exp["config"].get("eager"):
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
### Add Custom Callbacks
exp["config"]["callbacks"] = CustomCallbacks
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address)
else:
ray.init(address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
# NOTE: customs
for exp in experiments.values():
exp["loggers"] = make_loggers(args)
# launch training
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
verbose=verbose,
concurrent=True)
def test_connect_with_disconnected_node(shutdown_only):
config = {
"num_heartbeats_timeout": 50,
"raylet_heartbeat_timeout_milliseconds": 10,
}
cluster = Cluster()
cluster.add_node(num_cpus=0, _system_config=config)
ray.init(address=cluster.address)
p = init_error_pubsub()
errors = get_error_message(p, 1, timeout=5)
assert len(errors) == 0
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0)
cluster.remove_node(dead_node, allow_graceful=False)
errors = get_error_message(p, 1, ray_constants.REMOVED_NODE_ERROR)
assert len(errors) == 1
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0)
cluster.remove_node(dead_node, allow_graceful=False)
errors = get_error_message(p, 1, ray_constants.REMOVED_NODE_ERROR)
assert len(errors) == 1
# This node is killed by SIGTERM, ray_monitor will not mark it again.
removing_node = cluster.add_node(num_cpus=0)
cluster.remove_node(removing_node, allow_graceful=True)
errors = get_error_message(p, 1, timeout=2)
assert len(errors) == 0
# There is no connection error to a dead node.
errors = get_error_message(p, 1, timeout=2)
assert len(errors) == 0
p.close()
