def test_for_each_concur_sync(ray_start_regular_shared):
main_wait = Semaphore.remote(value=0)
test_wait = Semaphore.remote(value=0)
def task(x):
i, main_wait, test_wait = x
ray.get(main_wait.release.remote())
ray.get(test_wait.acquire.remote())
return i + 10
@ray.remote(num_cpus=0.01)
def to_list(it):
return list(it)
it = from_items([(i, main_wait, test_wait) for i in range(8)],
num_shards=2)
it = it.for_each(task, max_concurrency=2, resources={"num_cpus": 0.01})
list_promise = to_list.remote(it.gather_sync())
for i in range(4):
assert i in [0, 1, 2, 3]
ray.get(main_wait.acquire.remote())
# There should be exactly 4 tasks executing at this point.
assert ray.get(main_wait.locked.remote()) is True, "Too much parallelism"
for i in range(8):
ray.get(test_wait.release.remote())
assert repr(
it) == "ParallelIterator[from_items[tuple, 8, shards=2].for_each()]"
result_list = ray.get(list_promise)
assert set(result_list) == set(range(10, 18))
def test_limit_concurrency(shutdown_only):
ray.init(num_cpus=1)
block_task = Semaphore.remote(0)
block_driver = Semaphore.remote(0)
ray.get([block_task.locked.remote(), block_driver.locked.remote()])
@ray.remote(num_cpus=1)
def foo():
ray.get(block_driver.release.remote())
ray.get(block_task.acquire.remote())
refs = [foo.remote() for _ in range(20)]
block_driver_refs = [block_driver.acquire.remote() for _ in range(20)]
# Some of the tasks will run since we relax the cap, but not all because it
# should take exponentially long for the cap to be increased.
ready, not_ready = ray.wait(block_driver_refs, timeout=10, num_returns=20)
assert len(not_ready) >= 1
# Now the first instance of foo finishes, so the second starts to run.
ray.get([block_task.release.remote() for _ in range(19)])
ready, not_ready = ray.wait(block_driver_refs, timeout=10, num_returns=20)
assert len(not_ready) == 0
ready, not_ready = ray.wait(refs, num_returns=20, timeout=15)
assert len(ready) == 19
assert len(not_ready) == 1
def test_hybrid_policy(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 2
num_cpus = 10
for _ in range(num_nodes):
cluster.add_node(num_cpus=num_cpus, memory=num_cpus)
cluster.wait_for_nodes()
ray.init(address=cluster.address)
# `block_task` ensures that scheduled tasks do not return until all are
# running.
block_task = Semaphore.remote(0)
# `block_driver` ensures that the driver does not allow tasks to continue
# until all are running.
block_driver = Semaphore.remote(0)
# Add the memory resource because the cpu will be released in the ray.get
@ray.remote(num_cpus=1, memory=1)
def get_node():
ray.get(block_driver.release.remote())
ray.get(block_task.acquire.remote())
return ray.worker.global_worker.current_node_id
# Below the hybrid threshold we pack on the local node first.
refs = [get_node.remote() for _ in range(5)]
ray.get([block_driver.acquire.remote() for _ in refs])
ray.get([block_task.release.remote() for _ in refs])
nodes = ray.get(refs)
assert len(set(nodes)) == 1
# We pack the second node to the hybrid threshold.
refs = [get_node.remote() for _ in range(10)]
ray.get([block_driver.acquire.remote() for _ in refs])
ray.get([block_task.release.remote() for _ in refs])
nodes = ray.get(refs)
counter = collections.Counter(nodes)
for node_id in counter:
print(f"{node_id}: {counter[node_id]}")
assert counter[node_id] == 5
# Once all nodes are past the hybrid threshold we round robin.
# TODO (Alex): Ideally we could schedule less than 20 nodes here, but the
# policy is imperfect if a resource report interrupts the process.
refs = [get_node.remote() for _ in range(20)]
ray.get([block_driver.acquire.remote() for _ in refs])
ray.get([block_task.release.remote() for _ in refs])
nodes = ray.get(refs)
counter = collections.Counter(nodes)
for node_id in counter:
print(f"{node_id}: {counter[node_id]}")
assert counter[node_id] == 10, counter
def test_worker_failed(ray_start_workers_separate_multinode):
num_nodes, num_initial_workers = ray_start_workers_separate_multinode
block_worker = Semaphore.remote(0)
block_driver = Semaphore.remote(0)
ray.get([block_worker.locked.remote(), block_driver.locked.remote()])
# Acquire a custom resource that isn't released on `ray.get` to make sure
# this task gets spread across all the nodes.
@ray.remote(num_cpus=1, resources={"custom": 1})
def get_pids():
ray.get(block_driver.release.remote())
ray.get(block_worker.acquire.remote())
return os.getpid()
total_num_workers = num_nodes * num_initial_workers
pid_refs = [get_pids.remote() for _ in range(total_num_workers)]
ray.get([block_driver.acquire.remote() for _ in range(total_num_workers)])
ray.get([block_worker.release.remote() for _ in range(total_num_workers)])
pids = set(ray.get(pid_refs))
@ray.remote
def f(x):
time.sleep(0.5)
return x
# Submit more tasks than there are workers so that all workers and
# cores are utilized.
object_refs = [f.remote(i) for i in range(num_initial_workers * num_nodes)]
object_refs += [f.remote(object_ref) for object_ref in object_refs]
# Allow the tasks some time to begin executing.
time.sleep(0.1)
# Kill the workers as the tasks execute.
for pid in pids:
try:
os.kill(pid, SIGKILL)
except OSError:
# The process may have already exited due to worker capping.
pass
time.sleep(0.1)
# Make sure that we either get the object or we get an appropriate
# exception.
for object_ref in object_refs:
try:
ray.get(object_ref)
except (ray.exceptions.RayTaskError,
ray.exceptions.WorkerCrashedError):
pass
def test_warning_for_too_many_nested_tasks(shutdown_only):
# Check that if we run a workload which requires too many workers to be
# started that we will receive a warning.
num_cpus = 2
ray.init(num_cpus=num_cpus)
p = init_error_pubsub()
remote_wait = Semaphore.remote(value=0)
nested_wait = Semaphore.remote(value=0)
ray.get([
remote_wait.locked.remote(),
nested_wait.locked.remote(),
])
@ray.remote(num_cpus=0.25)
def f():
time.sleep(1000)
return 1
@ray.remote(num_cpus=0.25)
def h(nested_waits):
nested_wait.release.remote()
ray.get(nested_waits)
ray.get(f.remote())
@ray.remote(num_cpus=0.25)
def g(remote_waits, nested_waits):
# Sleep so that the f tasks all get submitted to the scheduler after
# the g tasks.
remote_wait.release.remote()
# wait until every lock is released.
ray.get(remote_waits)
ray.get(h.remote(nested_waits))
num_root_tasks = num_cpus * 4
# Lock remote task until everything is scheduled.
remote_waits = []
nested_waits = []
for _ in range(num_root_tasks):
remote_waits.append(remote_wait.acquire.remote())
nested_waits.append(nested_wait.acquire.remote())
[g.remote(remote_waits, nested_waits) for _ in range(num_root_tasks)]
errors = get_error_message(p, 1, ray_constants.WORKER_POOL_LARGE_ERROR)
assert len(errors) == 1
assert errors[0].type == ray_constants.WORKER_POOL_LARGE_ERROR
p.close()
def test_many_queued_tasks():
sema = Semaphore.remote(0)
@ray.remote(num_cpus=1)
def block():
ray.get(sema.acquire.remote())
@ray.remote(num_cpus=1)
def f():
pass
num_cpus = int(ray.cluster_resources()["CPU"])
blocked_tasks = []
for _ in range(num_cpus):
blocked_tasks.append(block.remote())
print("Submitting many tasks")
pending_tasks = []
for _ in trange(MAX_QUEUED_TASKS):
pending_tasks.append(f.remote())
# Make sure all the tasks can actually run.
for _ in range(num_cpus):
sema.release.remote()
print("Unblocking tasks")
for ref in tqdm(pending_tasks):
assert ray.get(ref) is None
def test_task_status(ray_start_regular):
address = ray_start_regular["address"]
@ray.remote
def dep(sema, x=None):
ray.get(sema.acquire.remote())
return
@ray.remote(num_gpus=1)
def impossible():
pass
# Filter out actor handle refs.
def filtered_summary():
data = "\n".join(
[
line
for line in memory_summary(address, line_wrap=False).split("\n")
if "ACTOR_HANDLE" not in line
]
)
print(data)
return data
sema = Semaphore.remote(value=0)
x = dep.remote(sema)
y = dep.remote(sema, x=x)
im = impossible.remote() # noqa
# x and its semaphore task are scheduled. im cannot
# be scheduled, so it is pending forever.
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_EXECUTION) == 2)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 1)
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 1)
z = dep.remote(sema, x=x)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 2)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_EXECUTION) == 2)
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 0)
sema.release.remote()
time.sleep(2)
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 1)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 0)
# y, z, and two semaphore tasks are scheduled.
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_EXECUTION) == 4)
sema.release.remote()
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 2)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 0)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_EXECUTION) == 2)
sema.release.remote()
ray.get(y)
ray.get(z)
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 3)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 0)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_EXECUTION) == 0)
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 1)
def test_zero_cpu_scheduling(shutdown_only):
ray.init(num_cpus=1)
block_task = Semaphore.remote(0)
block_driver = Semaphore.remote(0)
@ray.remote(num_cpus=0)
def foo():
ray.get(block_driver.release.remote())
ray.get(block_task.acquire.remote())
foo.remote()
foo.remote()
ray.get(block_driver.acquire.remote())
block_driver_ref = block_driver.acquire.remote()
# Both tasks should be running, so the driver should be unblocked.
ready, not_ready = ray.wait([block_driver_ref], timeout=1)
assert len(not_ready) == 0
def test_task_status(ray_start_regular):
address = ray_start_regular["address"]
@ray.remote
def dep(sema, x=None):
ray.get(sema.acquire.remote())
return
# Filter out actor handle refs.
def filtered_summary():
return "\n".join(
[
line
for line in memory_summary(address, line_wrap=False).split("\n")
if "ACTOR_HANDLE" not in line
]
)
sema = Semaphore.remote(value=0)
x = dep.remote(sema)
y = dep.remote(sema, x=x)
# x and its semaphore task are scheduled.
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 2)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 1)
z = dep.remote(sema, x=x)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 2)
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 2)
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 0)
sema.release.remote()
time.sleep(2)
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 1)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 0)
# y, z, and two semaphore tasks are scheduled.
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 4)
sema.release.remote()
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 2)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 0)
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 2)
sema.release.remote()
ray.get(y)
ray.get(z)
wait_for_condition(lambda: count(filtered_summary(), FINISHED) == 3)
wait_for_condition(lambda: count(filtered_summary(), WAITING_FOR_DEPENDENCIES) == 0)
wait_for_condition(lambda: count(filtered_summary(), SCHEDULED) == 0)